2024-11-26 21:52:02,025 main DEBUG Apache Log4j Core 2.17.2 initializing configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@15a34df2 2024-11-26 21:52:02,046 main DEBUG Took 0.018755 seconds to load 1 plugins from package org.apache.hadoop.hbase.logging 2024-11-26 21:52:02,047 main DEBUG PluginManager 'Core' found 129 plugins 2024-11-26 21:52:02,047 main DEBUG PluginManager 'Level' found 0 plugins 2024-11-26 21:52:02,048 main DEBUG PluginManager 'Lookup' found 16 plugins 2024-11-26 21:52:02,049 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-26 21:52:02,058 main DEBUG PluginManager 'TypeConverter' found 26 plugins 2024-11-26 21:52:02,074 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.util.MBeans", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-26 21:52:02,075 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-26 21:52:02,076 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.logging.TestJul2Slf4j", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-26 21:52:02,077 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-26 21:52:02,078 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.zookeeper", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-26 21:52:02,078 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-26 21:52:02,080 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSinkAdapter", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-26 21:52:02,080 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-26 21:52:02,081 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSystemImpl", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-26 21:52:02,081 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-26 21:52:02,083 main DEBUG LoggerConfig$Builder(additivity="false", level="WARN", levelAndRefs="null", name="org.apache.directory", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-26 21:52:02,083 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-26 21:52:02,084 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.ipc.FailedServers", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-26 21:52:02,084 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-26 21:52:02,085 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsConfig", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-26 21:52:02,085 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-26 21:52:02,086 main DEBUG LoggerConfig$Builder(additivity="null", level="INFO", levelAndRefs="null", name="org.apache.hadoop.hbase.ScheduledChore", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-26 21:52:02,086 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-26 21:52:02,087 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.regionserver.RSRpcServices", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-26 21:52:02,087 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-26 21:52:02,088 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-26 21:52:02,088 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-26 21:52:02,089 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-26 21:52:02,089 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-26 21:52:02,090 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hbase.thirdparty.io.netty.channel", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-26 21:52:02,090 main DEBUG Building Plugin[name=root, class=org.apache.logging.log4j.core.config.LoggerConfig$RootLogger]. 2024-11-26 21:52:02,092 main DEBUG LoggerConfig$RootLogger$Builder(additivity="null", level="null", levelAndRefs="INFO,Console", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-26 21:52:02,093 main DEBUG Building Plugin[name=loggers, class=org.apache.logging.log4j.core.config.LoggersPlugin]. 2024-11-26 21:52:02,095 main DEBUG createLoggers(={org.apache.hadoop.metrics2.util.MBeans, org.apache.hadoop.hbase.logging.TestJul2Slf4j, org.apache.zookeeper, org.apache.hadoop.metrics2.impl.MetricsSinkAdapter, org.apache.hadoop.metrics2.impl.MetricsSystemImpl, org.apache.directory, org.apache.hadoop.hbase.ipc.FailedServers, org.apache.hadoop.metrics2.impl.MetricsConfig, org.apache.hadoop.hbase.ScheduledChore, org.apache.hadoop.hbase.regionserver.RSRpcServices, org.apache.hadoop, org.apache.hadoop.hbase, org.apache.hbase.thirdparty.io.netty.channel, root}) 2024-11-26 21:52:02,096 main DEBUG Building Plugin[name=layout, class=org.apache.logging.log4j.core.layout.PatternLayout]. 2024-11-26 21:52:02,098 main DEBUG PatternLayout$Builder(pattern="%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n", PatternSelector=null, Configuration(PropertiesConfig), Replace=null, charset="null", alwaysWriteExceptions="null", disableAnsi="null", noConsoleNoAnsi="null", header="null", footer="null") 2024-11-26 21:52:02,098 main DEBUG PluginManager 'Converter' found 47 plugins 2024-11-26 21:52:02,108 main DEBUG Building Plugin[name=appender, class=org.apache.hadoop.hbase.logging.HBaseTestAppender]. 2024-11-26 21:52:02,112 main DEBUG HBaseTestAppender$Builder(target="SYSTEM_ERR", maxSize="1G", bufferedIo="null", bufferSize="null", immediateFlush="null", ignoreExceptions="null", PatternLayout(%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n), name="Console", Configuration(PropertiesConfig), Filter=null, ={}) 2024-11-26 21:52:02,114 main DEBUG Starting HBaseTestOutputStreamManager SYSTEM_ERR 2024-11-26 21:52:02,114 main DEBUG Building Plugin[name=appenders, class=org.apache.logging.log4j.core.config.AppendersPlugin]. 2024-11-26 21:52:02,115 main DEBUG createAppenders(={Console}) 2024-11-26 21:52:02,116 main DEBUG Configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@15a34df2 initialized 2024-11-26 21:52:02,116 main DEBUG Starting configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@15a34df2 2024-11-26 21:52:02,116 main DEBUG Started configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@15a34df2 OK. 2024-11-26 21:52:02,117 main DEBUG Shutting down OutputStreamManager SYSTEM_OUT.false.false-1 2024-11-26 21:52:02,118 main DEBUG OutputStream closed 2024-11-26 21:52:02,118 main DEBUG Shut down OutputStreamManager SYSTEM_OUT.false.false-1, all resources released: true 2024-11-26 21:52:02,119 main DEBUG Appender DefaultConsole-1 stopped with status true 2024-11-26 21:52:02,119 main DEBUG Stopped org.apache.logging.log4j.core.config.DefaultConfiguration@4f2b503c OK 2024-11-26 21:52:02,191 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6 2024-11-26 21:52:02,193 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=StatusLogger 2024-11-26 21:52:02,194 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=ContextSelector 2024-11-26 21:52:02,195 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name= 2024-11-26 21:52:02,196 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.directory 2024-11-26 21:52:02,196 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSinkAdapter 2024-11-26 21:52:02,196 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.zookeeper 2024-11-26 21:52:02,197 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.logging.TestJul2Slf4j 2024-11-26 21:52:02,197 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSystemImpl 2024-11-26 21:52:02,198 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.util.MBeans 2024-11-26 21:52:02,198 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase 2024-11-26 21:52:02,198 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop 2024-11-26 21:52:02,199 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ipc.FailedServers 2024-11-26 21:52:02,199 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.regionserver.RSRpcServices 2024-11-26 21:52:02,199 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsConfig 2024-11-26 21:52:02,199 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hbase.thirdparty.io.netty.channel 2024-11-26 21:52:02,200 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ScheduledChore 2024-11-26 21:52:02,201 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Appenders,name=Console 2024-11-26 21:52:02,203 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-26 21:52:02,203 main DEBUG Reconfiguration complete for context[name=1dbd16a6] at URI jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-logging/target/hbase-logging-4.0.0-alpha-1-SNAPSHOT-tests.jar!/log4j2.properties (org.apache.logging.log4j.core.LoggerContext@31c7528f) with optional ClassLoader: null 2024-11-26 21:52:02,203 main DEBUG Shutdown hook enabled. Registering a new one. 2024-11-26 21:52:02,204 main DEBUG LoggerContext[name=1dbd16a6, org.apache.logging.log4j.core.LoggerContext@31c7528f] started OK. 2024-11-26T21:52:02,216 INFO [main {}] hbase.HBaseClassTestRule(94): Test class org.apache.hadoop.hbase.io.hfile.TestBlockEvictionOnRegionMovement timeout: 13 mins 2024-11-26 21:52:02,219 main DEBUG AsyncLogger.ThreadNameStrategy=UNCACHED (user specified null, default is UNCACHED) 2024-11-26 21:52:02,220 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-26T21:52:02,454 DEBUG [main {}] hbase.HBaseTestingUtil(323): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7430f913-2694-54cf-261c-654b69c5fe06 2024-11-26T21:52:02,481 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: io.hfile.TestBlockEvictionOnRegionMovement#testBlockEvictionOnRegionMove Thread=12, OpenFileDescriptor=287, MaxFileDescriptor=1048576, SystemLoadAverage=145, ProcessCount=11, AvailableMemoryMB=11286 2024-11-26T21:52:02,572 WARN [Time-limited test {}] util.NativeCodeLoader(60): Unable to load native-hadoop library for your platform... using builtin-java classes where applicable 2024-11-26T21:52:02,657 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-26T21:52:02,673 INFO [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7430f913-2694-54cf-261c-654b69c5fe06/cluster_24567b58-1d07-1415-c32e-a7d969893315, deleteOnExit=true 2024-11-26T21:52:02,771 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7430f913-2694-54cf-261c-654b69c5fe06/cluster_24567b58-1d07-1415-c32e-a7d969893315/zookeeper_0, clientPort=64198, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7430f913-2694-54cf-261c-654b69c5fe06/cluster_24567b58-1d07-1415-c32e-a7d969893315/zookeeper_0/version-2, dataDirSize=457 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7430f913-2694-54cf-261c-654b69c5fe06/cluster_24567b58-1d07-1415-c32e-a7d969893315/zookeeper_0/version-2, dataLogSize=457 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0 2024-11-26T21:52:02,783 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=64198 2024-11-26T21:52:02,784 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-26T21:52:02,861 INFO [Time-limited test {}] util.FSUtils(489): Created version file at /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7430f913-2694-54cf-261c-654b69c5fe06/de866367-466a-3788-a91b-60c9e5aa6ab2 with version=8 2024-11-26T21:52:02,861 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1139): Setting hbase.fs.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7430f913-2694-54cf-261c-654b69c5fe06/de866367-466a-3788-a91b-60c9e5aa6ab2/hbase-staging 2024-11-26T21:52:02,952 DEBUG [Time-limited test {}] channel.MultithreadEventLoopGroup(44): -Dio.netty.eventLoopThreads: 16 2024-11-26T21:52:03,184 INFO [Time-limited test {}] client.ConnectionUtils(128): master/828b3f090c91:0 server-side Connection retries=45 2024-11-26T21:52:03,194 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-26T21:52:03,195 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-26T21:52:03,204 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-26T21:52:03,204 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-26T21:52:03,204 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-26T21:52:03,334 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService 2024-11-26T21:52:03,415 WARN [Time-limited test {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-26T21:52:03,453 INFO [Time-limited test {}] metrics.MetricRegistriesLoader(60): Loaded MetricRegistries class org.apache.hadoop.hbase.metrics.impl.MetricRegistriesImpl 2024-11-26T21:52:03,463 DEBUG [Time-limited test {}] util.ClassSize(228): Using Unsafe to estimate memory layout 2024-11-26T21:52:03,467 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-26T21:52:03,504 DEBUG [Time-limited test {}] channel.DefaultChannelId(84): -Dio.netty.processId: 92675 (auto-detected) 2024-11-26T21:52:03,505 DEBUG [Time-limited test {}] channel.DefaultChannelId(106): -Dio.netty.machineId: 02:42:ac:ff:fe:11:00:02 (auto-detected) 2024-11-26T21:52:03,528 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:36093 2024-11-26T21:52:03,556 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:36093 connecting to ZooKeeper ensemble=127.0.0.1:64198 2024-11-26T21:52:03,586 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:360930x0, quorum=127.0.0.1:64198, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-26T21:52:03,589 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:36093-0x1007e051f2e0000 connected 2024-11-26T21:52:03,617 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-26T21:52:03,619 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-26T21:52:03,627 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:36093-0x1007e051f2e0000, quorum=127.0.0.1:64198, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-26T21:52:03,631 INFO [Time-limited test {}] master.HMaster(525): hbase.rootdir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7430f913-2694-54cf-261c-654b69c5fe06/de866367-466a-3788-a91b-60c9e5aa6ab2, hbase.cluster.distributed=false 2024-11-26T21:52:03,655 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:36093-0x1007e051f2e0000, quorum=127.0.0.1:64198, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-26T21:52:03,660 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=36093 2024-11-26T21:52:03,661 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=36093 2024-11-26T21:52:03,661 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=36093 2024-11-26T21:52:03,662 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=36093 2024-11-26T21:52:03,662 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=36093 2024-11-26T21:52:03,741 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/828b3f090c91:0 server-side Connection retries=45 2024-11-26T21:52:03,743 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-26T21:52:03,744 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-26T21:52:03,744 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-26T21:52:03,744 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-26T21:52:03,745 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-26T21:52:03,748 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-26T21:52:03,751 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-26T21:52:03,752 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:35731 2024-11-26T21:52:03,754 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:35731 connecting to ZooKeeper ensemble=127.0.0.1:64198 2024-11-26T21:52:03,755 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-26T21:52:03,756 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-26T21:52:03,759 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:357310x0, quorum=127.0.0.1:64198, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-26T21:52:03,759 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:35731-0x1007e051f2e0001, quorum=127.0.0.1:64198, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-26T21:52:03,759 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:35731-0x1007e051f2e0001 connected 2024-11-26T21:52:03,765 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-26T21:52:03,779 INFO [Time-limited test {}] bucket.FileIOEngine(99): Allocating cache 400 MB, on the path:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7430f913-2694-54cf-261c-654b69c5fe06/bucket.cache 2024-11-26T21:52:03,788 INFO [Time-limited test {}] bucket.BucketCache(364): Instantiating BucketCache with acceptableFactor: 0.95, minFactor: 0.85, extraFreeFactor: 0.1, singleFactor: 0.25, multiFactor: 0.5, memoryFactor: 0.25, useStrongRef: false 2024-11-26T21:52:03,790 INFO [Time-limited test {}] bucket.BucketCache(478): Starting BucketCachePersisterThread 2024-11-26T21:52:03,792 INFO [Time-limited test {}] bucket.BucketCachePersister(38): BucketCachePersister started with interval: 100 2024-11-26T21:52:03,795 INFO [Thread-10 {}] bucket.BucketCache(1530): Started retrieving bucket cache from file 2024-11-26T21:52:03,795 WARN [Thread-10 {}] bucket.BucketCache(1533): Persistence file missing! It's ok if it's first run after enabling persistent cache. 2024-11-26T21:52:03,802 INFO [Time-limited test {}] bucket.BucketCache(403): Started bucket cache; ioengine=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7430f913-2694-54cf-261c-654b69c5fe06/bucket.cache, capacity=400 MB, blockSize=64 KB, writerThreadNum=3, writerQLen=64, persistencePath=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7430f913-2694-54cf-261c-654b69c5fe06/bucket.persistence, bucketAllocator=org.apache.hadoop.hbase.io.hfile.bucket.BucketAllocator 2024-11-26T21:52:03,806 INFO [Thread-10 {}] bucket.BucketAllocator(343): Cache totalSize=418148352, buckets=199, bucket capacity=2101248=(4*525312)=(FEWEST_ITEMS_IN_BUCKET*(largest configured bucketcache size)) 2024-11-26T21:52:03,806 INFO [Thread-10 {}] bucket.BucketCache(413): Persistent bucket cache recovery from /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7430f913-2694-54cf-261c-654b69c5fe06/bucket.persistence is complete. 2024-11-26T21:52:03,809 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-26T21:52:03,813 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:35731-0x1007e051f2e0001, quorum=127.0.0.1:64198, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-26T21:52:03,820 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:35731-0x1007e051f2e0001, quorum=127.0.0.1:64198, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-26T21:52:03,821 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=35731 2024-11-26T21:52:03,822 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=35731 2024-11-26T21:52:03,822 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=35731 2024-11-26T21:52:03,823 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=35731 2024-11-26T21:52:03,824 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=35731 2024-11-26T21:52:03,849 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/828b3f090c91:0 server-side Connection retries=45 2024-11-26T21:52:03,849 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-26T21:52:03,850 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-26T21:52:03,850 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-26T21:52:03,851 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-26T21:52:03,851 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-26T21:52:03,851 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-26T21:52:03,852 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-26T21:52:03,853 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:39637 2024-11-26T21:52:03,855 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:39637 connecting to ZooKeeper ensemble=127.0.0.1:64198 2024-11-26T21:52:03,855 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-26T21:52:03,856 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-26T21:52:03,863 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:396370x0, quorum=127.0.0.1:64198, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-26T21:52:03,864 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:396370x0, quorum=127.0.0.1:64198, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-26T21:52:03,864 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:39637-0x1007e051f2e0002 connected 2024-11-26T21:52:03,865 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-26T21:52:03,867 INFO [Time-limited test {}] bucket.FileIOEngine(99): Allocating cache 400 MB, on the path:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7430f913-2694-54cf-261c-654b69c5fe06/bucket.cache 2024-11-26T21:52:03,867 INFO [Time-limited test {}] bucket.BucketCache(364): Instantiating BucketCache with acceptableFactor: 0.95, minFactor: 0.85, extraFreeFactor: 0.1, singleFactor: 0.25, multiFactor: 0.5, memoryFactor: 0.25, useStrongRef: false 2024-11-26T21:52:03,868 INFO [Time-limited test {}] bucket.BucketCache(478): Starting BucketCachePersisterThread 2024-11-26T21:52:03,868 INFO [Time-limited test {}] bucket.BucketCachePersister(38): BucketCachePersister started with interval: 100 2024-11-26T21:52:03,869 INFO [Thread-11 {}] bucket.BucketCache(1530): Started retrieving bucket cache from file 2024-11-26T21:52:03,869 WARN [Thread-11 {}] bucket.BucketCache(1533): Persistence file missing! It's ok if it's first run after enabling persistent cache. 2024-11-26T21:52:03,870 INFO [Time-limited test {}] bucket.BucketCache(403): Started bucket cache; ioengine=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7430f913-2694-54cf-261c-654b69c5fe06/bucket.cache, capacity=400 MB, blockSize=64 KB, writerThreadNum=3, writerQLen=64, persistencePath=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7430f913-2694-54cf-261c-654b69c5fe06/bucket.persistence, bucketAllocator=org.apache.hadoop.hbase.io.hfile.bucket.BucketAllocator 2024-11-26T21:52:03,870 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-26T21:52:03,870 INFO [Thread-11 {}] bucket.BucketAllocator(343): Cache totalSize=418148352, buckets=199, bucket capacity=2101248=(4*525312)=(FEWEST_ITEMS_IN_BUCKET*(largest configured bucketcache size)) 2024-11-26T21:52:03,870 INFO [Thread-11 {}] bucket.BucketCache(413): Persistent bucket cache recovery from /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7430f913-2694-54cf-261c-654b69c5fe06/bucket.persistence is complete. 2024-11-26T21:52:03,872 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:39637-0x1007e051f2e0002, quorum=127.0.0.1:64198, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-26T21:52:03,874 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:39637-0x1007e051f2e0002, quorum=127.0.0.1:64198, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-26T21:52:03,877 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=39637 2024-11-26T21:52:03,878 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=39637 2024-11-26T21:52:03,880 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=39637 2024-11-26T21:52:03,881 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=39637 2024-11-26T21:52:03,882 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=39637 2024-11-26T21:52:03,902 DEBUG [M:0;828b3f090c91:36093 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;828b3f090c91:36093 2024-11-26T21:52:03,905 INFO [master/828b3f090c91:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/828b3f090c91,36093,1732657923024 2024-11-26T21:52:03,914 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36093-0x1007e051f2e0000, quorum=127.0.0.1:64198, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-26T21:52:03,914 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35731-0x1007e051f2e0001, quorum=127.0.0.1:64198, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-26T21:52:03,914 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39637-0x1007e051f2e0002, quorum=127.0.0.1:64198, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-26T21:52:03,917 DEBUG [master/828b3f090c91:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:36093-0x1007e051f2e0000, quorum=127.0.0.1:64198, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/828b3f090c91,36093,1732657923024 2024-11-26T21:52:03,949 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39637-0x1007e051f2e0002, quorum=127.0.0.1:64198, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-26T21:52:03,949 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35731-0x1007e051f2e0001, quorum=127.0.0.1:64198, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-26T21:52:03,949 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36093-0x1007e051f2e0000, quorum=127.0.0.1:64198, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-26T21:52:03,950 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39637-0x1007e051f2e0002, quorum=127.0.0.1:64198, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-26T21:52:03,950 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35731-0x1007e051f2e0001, quorum=127.0.0.1:64198, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-26T21:52:03,952 DEBUG [master/828b3f090c91:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:36093-0x1007e051f2e0000, quorum=127.0.0.1:64198, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-11-26T21:52:03,954 INFO [master/828b3f090c91:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/828b3f090c91,36093,1732657923024 from backup master directory 2024-11-26T21:52:03,957 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35731-0x1007e051f2e0001, quorum=127.0.0.1:64198, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-26T21:52:03,957 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36093-0x1007e051f2e0000, quorum=127.0.0.1:64198, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/828b3f090c91,36093,1732657923024 2024-11-26T21:52:03,957 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39637-0x1007e051f2e0002, quorum=127.0.0.1:64198, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-26T21:52:03,957 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36093-0x1007e051f2e0000, quorum=127.0.0.1:64198, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-26T21:52:03,958 WARN [master/828b3f090c91:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-26T21:52:03,958 INFO [master/828b3f090c91:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=828b3f090c91,36093,1732657923024 2024-11-26T21:52:03,961 INFO [master/828b3f090c91:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating data MemStoreChunkPool with chunk size 2 MB, max count 396, initial count 0 2024-11-26T21:52:03,963 INFO [master/828b3f090c91:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating index MemStoreChunkPool with chunk size 204.80 KB, max count 440, initial count 0 2024-11-26T21:52:03,974 DEBUG [master/828b3f090c91:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7430f913-2694-54cf-261c-654b69c5fe06/de866367-466a-3788-a91b-60c9e5aa6ab2/hbase.id] with ID: a0458ab8-5c44-4513-aa4c-756a7f2c1a4b 2024-11-26T21:52:03,974 DEBUG [master/828b3f090c91:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7430f913-2694-54cf-261c-654b69c5fe06/de866367-466a-3788-a91b-60c9e5aa6ab2/.tmp/hbase.id 2024-11-26T21:52:03,980 DEBUG [master/828b3f090c91:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7430f913-2694-54cf-261c-654b69c5fe06/de866367-466a-3788-a91b-60c9e5aa6ab2/.tmp/hbase.id]:[file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7430f913-2694-54cf-261c-654b69c5fe06/de866367-466a-3788-a91b-60c9e5aa6ab2/hbase.id] 2024-11-26T21:52:04,016 DEBUG [master/828b3f090c91:0:becomeActiveMaster {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-26T21:52:04,020 INFO [master/828b3f090c91:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem. 2024-11-26T21:52:04,041 INFO [master/828b3f090c91:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 17ms. 2024-11-26T21:52:04,044 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39637-0x1007e051f2e0002, quorum=127.0.0.1:64198, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-26T21:52:04,044 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35731-0x1007e051f2e0001, quorum=127.0.0.1:64198, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-26T21:52:04,044 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36093-0x1007e051f2e0000, quorum=127.0.0.1:64198, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-26T21:52:04,069 INFO [master/828b3f090c91:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-26T21:52:04,071 INFO [master/828b3f090c91:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-11-26T21:52:04,111 DEBUG [master/828b3f090c91:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(244): No decryptEncryptedDataEncryptionKey method in DFSClient, should be hadoop version with HDFS-12396 java.lang.NoSuchMethodException: org.apache.hadoop.hdfs.DFSClient.decryptEncryptedDataEncryptionKey(org.apache.hadoop.fs.FileEncryptionInfo) at java.lang.Class.getDeclaredMethod(Class.java:2675) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelperWithoutHDFS12396(FanOutOneBlockAsyncDFSOutputSaslHelper.java:183) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelper(FanOutOneBlockAsyncDFSOutputSaslHelper.java:242) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.(FanOutOneBlockAsyncDFSOutputSaslHelper.java:253) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.lang.Class.forName0(Native Method) ~[?:?] at java.lang.Class.forName(Class.java:375) ~[?:?] at org.apache.hadoop.hbase.wal.AsyncFSWALProvider.load(AsyncFSWALProvider.java:150) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.getProviderClass(WALFactory.java:174) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:262) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:231) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegion.create(MasterRegion.java:400) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegionFactory.create(MasterRegionFactory.java:135) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.finishActiveMasterInitialization(HMaster.java:1003) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.startActiveMasterManager(HMaster.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.lambda$run$0(HMaster.java:613) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.lambda$tracedRunnable$2(TraceUtil.java:155) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T21:52:04,116 INFO [master/828b3f090c91:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-26T21:52:04,166 INFO [master/828b3f090c91:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7430f913-2694-54cf-261c-654b69c5fe06/de866367-466a-3788-a91b-60c9e5aa6ab2/MasterData/data/master/store 2024-11-26T21:52:04,187 INFO [master/828b3f090c91:0:becomeActiveMaster {}] throttle.StoreHotnessProtector(112): StoreHotnessProtector is disabled. Set hbase.region.store.parallel.put.limit > 0 to enable, which may help mitigate load under heavy write pressure. 2024-11-26T21:52:04,190 DEBUG [master/828b3f090c91:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-26T21:52:04,192 DEBUG [master/828b3f090c91:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-26T21:52:04,192 INFO [master/828b3f090c91:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-26T21:52:04,192 DEBUG [master/828b3f090c91:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-26T21:52:04,194 DEBUG [master/828b3f090c91:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-26T21:52:04,194 DEBUG [master/828b3f090c91:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-26T21:52:04,194 INFO [master/828b3f090c91:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-26T21:52:04,195 DEBUG [master/828b3f090c91:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1732657924191Disabling compacts and flushes for region at 1732657924191Disabling writes for close at 1732657924194 (+3 ms)Writing region close event to WAL at 1732657924194Closed at 1732657924194 2024-11-26T21:52:04,196 WARN [master/828b3f090c91:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7430f913-2694-54cf-261c-654b69c5fe06/de866367-466a-3788-a91b-60c9e5aa6ab2/MasterData/data/master/store/.initializing 2024-11-26T21:52:04,196 DEBUG [master/828b3f090c91:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7430f913-2694-54cf-261c-654b69c5fe06/de866367-466a-3788-a91b-60c9e5aa6ab2/MasterData/WALs/828b3f090c91,36093,1732657923024 2024-11-26T21:52:04,208 INFO [master/828b3f090c91:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-26T21:52:04,220 INFO [master/828b3f090c91:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=828b3f090c91%2C36093%2C1732657923024, suffix=, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7430f913-2694-54cf-261c-654b69c5fe06/de866367-466a-3788-a91b-60c9e5aa6ab2/MasterData/WALs/828b3f090c91,36093,1732657923024, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7430f913-2694-54cf-261c-654b69c5fe06/de866367-466a-3788-a91b-60c9e5aa6ab2/MasterData/oldWALs, maxLogs=10 2024-11-26T21:52:04,256 INFO [master/828b3f090c91:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7430f913-2694-54cf-261c-654b69c5fe06/de866367-466a-3788-a91b-60c9e5aa6ab2/MasterData/WALs/828b3f090c91,36093,1732657923024/828b3f090c91%2C36093%2C1732657923024.1732657924224 2024-11-26T21:52:04,257 DEBUG [master/828b3f090c91:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-11-26T21:52:04,258 DEBUG [master/828b3f090c91:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-11-26T21:52:04,259 DEBUG [master/828b3f090c91:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-26T21:52:04,262 DEBUG [master/828b3f090c91:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-11-26T21:52:04,264 DEBUG [master/828b3f090c91:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-11-26T21:52:04,312 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-11-26T21:52:04,350 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-11-26T21:52:04,356 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T21:52:04,359 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-26T21:52:04,360 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-11-26T21:52:04,366 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-11-26T21:52:04,367 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T21:52:04,367 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-26T21:52:04,368 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-11-26T21:52:04,373 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-11-26T21:52:04,374 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T21:52:04,375 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-26T21:52:04,375 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-11-26T21:52:04,381 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-11-26T21:52:04,381 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T21:52:04,382 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-26T21:52:04,382 DEBUG [master/828b3f090c91:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682 2024-11-26T21:52:04,386 DEBUG [master/828b3f090c91:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7430f913-2694-54cf-261c-654b69c5fe06/de866367-466a-3788-a91b-60c9e5aa6ab2/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-26T21:52:04,387 DEBUG [master/828b3f090c91:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7430f913-2694-54cf-261c-654b69c5fe06/de866367-466a-3788-a91b-60c9e5aa6ab2/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-26T21:52:04,391 DEBUG [master/828b3f090c91:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682 2024-11-26T21:52:04,392 DEBUG [master/828b3f090c91:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682 2024-11-26T21:52:04,395 DEBUG [master/828b3f090c91:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-26T21:52:04,398 DEBUG [master/828b3f090c91:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-11-26T21:52:04,405 DEBUG [master/828b3f090c91:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7430f913-2694-54cf-261c-654b69c5fe06/de866367-466a-3788-a91b-60c9e5aa6ab2/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-26T21:52:04,405 INFO [master/828b3f090c91:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=69348570, jitterRate=0.03337422013282776}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-26T21:52:04,413 DEBUG [master/828b3f090c91:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682: Writing region info on filesystem at 1732657924277Initializing all the Stores at 1732657924277Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732657924278 (+1 ms)Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732657924279 (+1 ms)Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732657924280 (+1 ms)Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732657924280Cleaning up temporary data from old regions at 1732657924392 (+112 ms)Region opened successfully at 1732657924413 (+21 ms) 2024-11-26T21:52:04,414 INFO [master/828b3f090c91:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-11-26T21:52:04,445 DEBUG [master/828b3f090c91:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5696888d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=828b3f090c91/172.17.0.2:0 2024-11-26T21:52:04,477 INFO [master/828b3f090c91:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating... 2024-11-26T21:52:04,488 INFO [master/828b3f090c91:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5 2024-11-26T21:52:04,488 INFO [master/828b3f090c91:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50 2024-11-26T21:52:04,491 INFO [master/828b3f090c91:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-26T21:52:04,492 INFO [master/828b3f090c91:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-26T21:52:04,497 INFO [master/828b3f090c91:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 5 msec 2024-11-26T21:52:04,498 INFO [master/828b3f090c91:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-11-26T21:52:04,523 INFO [master/828b3f090c91:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2024-11-26T21:52:04,536 DEBUG [master/828b3f090c91:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:36093-0x1007e051f2e0000, quorum=127.0.0.1:64198, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error) 2024-11-26T21:52:04,538 DEBUG [master/828b3f090c91:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false 2024-11-26T21:52:04,541 INFO [master/828b3f090c91:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2024-11-26T21:52:04,544 DEBUG [master/828b3f090c91:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:36093-0x1007e051f2e0000, quorum=127.0.0.1:64198, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error) 2024-11-26T21:52:04,545 DEBUG [master/828b3f090c91:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false 2024-11-26T21:52:04,548 INFO [master/828b3f090c91:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2024-11-26T21:52:04,554 DEBUG [master/828b3f090c91:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:36093-0x1007e051f2e0000, quorum=127.0.0.1:64198, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error) 2024-11-26T21:52:04,555 DEBUG [master/828b3f090c91:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false 2024-11-26T21:52:04,557 DEBUG [master/828b3f090c91:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:36093-0x1007e051f2e0000, quorum=127.0.0.1:64198, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error) 2024-11-26T21:52:04,559 DEBUG [master/828b3f090c91:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false 2024-11-26T21:52:04,577 DEBUG [master/828b3f090c91:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:36093-0x1007e051f2e0000, quorum=127.0.0.1:64198, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error) 2024-11-26T21:52:04,578 DEBUG [master/828b3f090c91:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false 2024-11-26T21:52:04,589 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35731-0x1007e051f2e0001, quorum=127.0.0.1:64198, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-26T21:52:04,589 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39637-0x1007e051f2e0002, quorum=127.0.0.1:64198, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-26T21:52:04,590 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36093-0x1007e051f2e0000, quorum=127.0.0.1:64198, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-26T21:52:04,590 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35731-0x1007e051f2e0001, quorum=127.0.0.1:64198, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-26T21:52:04,590 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36093-0x1007e051f2e0000, quorum=127.0.0.1:64198, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-26T21:52:04,590 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39637-0x1007e051f2e0002, quorum=127.0.0.1:64198, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-26T21:52:04,593 INFO [master/828b3f090c91:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=828b3f090c91,36093,1732657923024, sessionid=0x1007e051f2e0000, setting cluster-up flag (Was=false) 2024-11-26T21:52:04,604 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35731-0x1007e051f2e0001, quorum=127.0.0.1:64198, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-26T21:52:04,604 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39637-0x1007e051f2e0002, quorum=127.0.0.1:64198, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-26T21:52:04,604 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36093-0x1007e051f2e0000, quorum=127.0.0.1:64198, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-26T21:52:04,608 DEBUG [master/828b3f090c91:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort 2024-11-26T21:52:04,611 DEBUG [master/828b3f090c91:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=828b3f090c91,36093,1732657923024 2024-11-26T21:52:04,615 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35731-0x1007e051f2e0001, quorum=127.0.0.1:64198, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-26T21:52:04,615 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36093-0x1007e051f2e0000, quorum=127.0.0.1:64198, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-26T21:52:04,615 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39637-0x1007e051f2e0002, quorum=127.0.0.1:64198, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-26T21:52:04,620 DEBUG [master/828b3f090c91:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort 2024-11-26T21:52:04,622 DEBUG [master/828b3f090c91:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=828b3f090c91,36093,1732657923024 2024-11-26T21:52:04,627 INFO [master/828b3f090c91:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7430f913-2694-54cf-261c-654b69c5fe06/de866367-466a-3788-a91b-60c9e5aa6ab2/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again 2024-11-26T21:52:04,687 INFO [RS:0;828b3f090c91:35731 {}] regionserver.HRegionServer(746): ClusterId : a0458ab8-5c44-4513-aa4c-756a7f2c1a4b 2024-11-26T21:52:04,687 INFO [RS:1;828b3f090c91:39637 {}] regionserver.HRegionServer(746): ClusterId : a0458ab8-5c44-4513-aa4c-756a7f2c1a4b 2024-11-26T21:52:04,690 DEBUG [RS:0;828b3f090c91:35731 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-26T21:52:04,690 DEBUG [RS:1;828b3f090c91:39637 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-26T21:52:04,695 DEBUG [RS:0;828b3f090c91:35731 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-26T21:52:04,695 DEBUG [RS:1;828b3f090c91:39637 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-26T21:52:04,695 DEBUG [RS:0;828b3f090c91:35731 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-26T21:52:04,695 DEBUG [RS:1;828b3f090c91:39637 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-26T21:52:04,698 DEBUG [RS:0;828b3f090c91:35731 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-26T21:52:04,698 DEBUG [RS:1;828b3f090c91:39637 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-26T21:52:04,699 DEBUG [RS:1;828b3f090c91:39637 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@30cf11c2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=828b3f090c91/172.17.0.2:0 2024-11-26T21:52:04,699 DEBUG [RS:0;828b3f090c91:35731 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5b4c1dca, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=828b3f090c91/172.17.0.2:0 2024-11-26T21:52:04,704 DEBUG [master/828b3f090c91:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-26T21:52:04,716 INFO [master/828b3f090c91:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2 2024-11-26T21:52:04,720 DEBUG [RS:0;828b3f090c91:35731 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;828b3f090c91:35731 2024-11-26T21:52:04,720 DEBUG [RS:1;828b3f090c91:39637 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:1;828b3f090c91:39637 2024-11-26T21:52:04,723 INFO [master/828b3f090c91:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2024-11-26T21:52:04,723 INFO [RS:0;828b3f090c91:35731 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-26T21:52:04,723 INFO [RS:1;828b3f090c91:39637 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-26T21:52:04,723 INFO [RS:1;828b3f090c91:39637 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-26T21:52:04,723 INFO [RS:0;828b3f090c91:35731 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-26T21:52:04,724 DEBUG [RS:1;828b3f090c91:39637 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-26T21:52:04,724 DEBUG [RS:0;828b3f090c91:35731 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-26T21:52:04,727 INFO [RS:0;828b3f090c91:35731 {}] regionserver.HRegionServer(2659): reportForDuty to master=828b3f090c91,36093,1732657923024 with port=35731, startcode=1732657923709 2024-11-26T21:52:04,727 INFO [RS:1;828b3f090c91:39637 {}] regionserver.HRegionServer(2659): reportForDuty to master=828b3f090c91,36093,1732657923024 with port=39637, startcode=1732657923848 2024-11-26T21:52:04,730 DEBUG [master/828b3f090c91:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: 828b3f090c91,36093,1732657923024 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2024-11-26T21:52:04,738 DEBUG [master/828b3f090c91:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/828b3f090c91:0, corePoolSize=5, maxPoolSize=5 2024-11-26T21:52:04,738 DEBUG [master/828b3f090c91:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/828b3f090c91:0, corePoolSize=5, maxPoolSize=5 2024-11-26T21:52:04,738 DEBUG [master/828b3f090c91:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/828b3f090c91:0, corePoolSize=5, maxPoolSize=5 2024-11-26T21:52:04,738 DEBUG [master/828b3f090c91:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/828b3f090c91:0, corePoolSize=5, maxPoolSize=5 2024-11-26T21:52:04,739 DEBUG [master/828b3f090c91:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/828b3f090c91:0, corePoolSize=10, maxPoolSize=10 2024-11-26T21:52:04,739 DEBUG [RS:0;828b3f090c91:35731 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-26T21:52:04,739 DEBUG [RS:1;828b3f090c91:39637 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-26T21:52:04,739 DEBUG [master/828b3f090c91:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/828b3f090c91:0, corePoolSize=1, maxPoolSize=1 2024-11-26T21:52:04,739 DEBUG [master/828b3f090c91:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/828b3f090c91:0, corePoolSize=2, maxPoolSize=2 2024-11-26T21:52:04,740 DEBUG [master/828b3f090c91:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/828b3f090c91:0, corePoolSize=1, maxPoolSize=1 2024-11-26T21:52:04,744 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-26T21:52:04,745 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region 2024-11-26T21:52:04,746 INFO [master/828b3f090c91:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732657954746 2024-11-26T21:52:04,748 INFO [master/828b3f090c91:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2024-11-26T21:52:04,749 INFO [master/828b3f090c91:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2024-11-26T21:52:04,750 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T21:52:04,751 INFO [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-11-26T21:52:04,753 INFO [master/828b3f090c91:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2024-11-26T21:52:04,753 INFO [master/828b3f090c91:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2024-11-26T21:52:04,754 INFO [master/828b3f090c91:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2024-11-26T21:52:04,754 INFO [master/828b3f090c91:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2024-11-26T21:52:04,762 INFO [master/828b3f090c91:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-26T21:52:04,766 INFO [master/828b3f090c91:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2024-11-26T21:52:04,768 INFO [master/828b3f090c91:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2024-11-26T21:52:04,768 INFO [master/828b3f090c91:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2024-11-26T21:52:04,773 INFO [master/828b3f090c91:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2024-11-26T21:52:04,774 INFO [master/828b3f090c91:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2024-11-26T21:52:04,776 DEBUG [master/828b3f090c91:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/828b3f090c91:0:becomeActiveMaster-HFileCleaner.large.0-1732657924775,5,FailOnTimeoutGroup] 2024-11-26T21:52:04,781 DEBUG [master/828b3f090c91:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/828b3f090c91:0:becomeActiveMaster-HFileCleaner.small.0-1732657924776,5,FailOnTimeoutGroup] 2024-11-26T21:52:04,781 INFO [master/828b3f090c91:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-26T21:52:04,782 INFO [master/828b3f090c91:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2024-11-26T21:52:04,782 INFO [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7430f913-2694-54cf-261c-654b69c5fe06/de866367-466a-3788-a91b-60c9e5aa6ab2/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321 2024-11-26T21:52:04,783 INFO [master/828b3f090c91:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2024-11-26T21:52:04,783 INFO [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7430f913-2694-54cf-261c-654b69c5fe06/de866367-466a-3788-a91b-60c9e5aa6ab2 2024-11-26T21:52:04,783 INFO [master/828b3f090c91:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2024-11-26T21:52:04,800 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-26T21:52:04,803 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family info of region 1588230740 2024-11-26T21:52:04,808 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-26T21:52:04,808 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T21:52:04,808 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-26T21:52:04,808 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family ns of region 1588230740 2024-11-26T21:52:04,812 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-26T21:52:04,812 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T21:52:04,813 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-26T21:52:04,813 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family rep_barrier of region 1588230740 2024-11-26T21:52:04,820 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-26T21:52:04,820 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T21:52:04,820 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-26T21:52:04,821 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family table of region 1588230740 2024-11-26T21:52:04,822 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35961, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=RegionServerStatusService 2024-11-26T21:52:04,822 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49305, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=RegionServerStatusService 2024-11-26T21:52:04,826 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-26T21:52:04,826 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T21:52:04,826 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-26T21:52:04,827 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-26T21:52:04,827 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7430f913-2694-54cf-261c-654b69c5fe06/de866367-466a-3788-a91b-60c9e5aa6ab2/data/hbase/meta/1588230740 2024-11-26T21:52:04,828 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7430f913-2694-54cf-261c-654b69c5fe06/de866367-466a-3788-a91b-60c9e5aa6ab2/data/hbase/meta/1588230740 2024-11-26T21:52:04,828 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-26T21:52:04,828 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-26T21:52:04,828 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-26T21:52:04,829 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-26T21:52:04,830 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=36093 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 828b3f090c91,39637,1732657923848 2024-11-26T21:52:04,833 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=36093 {}] master.ServerManager(517): Registering regionserver=828b3f090c91,39637,1732657923848 2024-11-26T21:52:04,839 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7430f913-2694-54cf-261c-654b69c5fe06/de866367-466a-3788-a91b-60c9e5aa6ab2/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-26T21:52:04,839 INFO [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=69262335, jitterRate=0.032089218497276306}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-26T21:52:04,843 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740: Writing region info on filesystem at 1732657924800Initializing all the Stores at 1732657924800Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732657924801 (+1 ms)Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732657924803 (+2 ms)Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732657924803Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732657924803Cleaning up temporary data from old regions at 1732657924828 (+25 ms)Region opened successfully at 1732657924843 (+15 ms) 2024-11-26T21:52:04,844 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-26T21:52:04,844 INFO [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-26T21:52:04,844 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-26T21:52:04,844 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-26T21:52:04,844 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-26T21:52:04,845 INFO [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-26T21:52:04,845 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=36093 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 828b3f090c91,35731,1732657923709 2024-11-26T21:52:04,845 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1732657924843Disabling compacts and flushes for region at 1732657924843Disabling writes for close at 1732657924844 (+1 ms)Writing region close event to WAL at 1732657924845 (+1 ms)Closed at 1732657924845 2024-11-26T21:52:04,846 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=36093 {}] master.ServerManager(517): Registering regionserver=828b3f090c91,35731,1732657923709 2024-11-26T21:52:04,848 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-26T21:52:04,848 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta 2024-11-26T21:52:04,850 DEBUG [RS:1;828b3f090c91:39637 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7430f913-2694-54cf-261c-654b69c5fe06/de866367-466a-3788-a91b-60c9e5aa6ab2 2024-11-26T21:52:04,850 DEBUG [RS:1;828b3f090c91:39637 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=file:/ 2024-11-26T21:52:04,850 DEBUG [RS:1;828b3f090c91:39637 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-26T21:52:04,852 DEBUG [RS:0;828b3f090c91:35731 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7430f913-2694-54cf-261c-654b69c5fe06/de866367-466a-3788-a91b-60c9e5aa6ab2 2024-11-26T21:52:04,852 DEBUG [RS:0;828b3f090c91:35731 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=file:/ 2024-11-26T21:52:04,852 DEBUG [RS:0;828b3f090c91:35731 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-26T21:52:04,855 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36093-0x1007e051f2e0000, quorum=127.0.0.1:64198, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-26T21:52:04,856 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-11-26T21:52:04,857 DEBUG [RS:1;828b3f090c91:39637 {}] zookeeper.ZKUtil(111): regionserver:39637-0x1007e051f2e0002, quorum=127.0.0.1:64198, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/828b3f090c91,39637,1732657923848 2024-11-26T21:52:04,857 WARN [RS:1;828b3f090c91:39637 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-26T21:52:04,857 DEBUG [RS:0;828b3f090c91:35731 {}] zookeeper.ZKUtil(111): regionserver:35731-0x1007e051f2e0001, quorum=127.0.0.1:64198, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/828b3f090c91,35731,1732657923709 2024-11-26T21:52:04,858 INFO [RS:1;828b3f090c91:39637 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-26T21:52:04,858 WARN [RS:0;828b3f090c91:35731 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-26T21:52:04,858 INFO [RS:0;828b3f090c91:35731 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-26T21:52:04,858 DEBUG [RS:1;828b3f090c91:39637 {}] regionserver.HRegionServer(1793): logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7430f913-2694-54cf-261c-654b69c5fe06/de866367-466a-3788-a91b-60c9e5aa6ab2/WALs/828b3f090c91,39637,1732657923848 2024-11-26T21:52:04,858 DEBUG [RS:0;828b3f090c91:35731 {}] regionserver.HRegionServer(1793): logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7430f913-2694-54cf-261c-654b69c5fe06/de866367-466a-3788-a91b-60c9e5aa6ab2/WALs/828b3f090c91,35731,1732657923709 2024-11-26T21:52:04,859 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [828b3f090c91,39637,1732657923848] 2024-11-26T21:52:04,859 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [828b3f090c91,35731,1732657923709] 2024-11-26T21:52:04,864 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-26T21:52:04,867 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2024-11-26T21:52:04,891 INFO [RS:1;828b3f090c91:39637 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-26T21:52:04,891 INFO [RS:0;828b3f090c91:35731 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-26T21:52:04,914 INFO [RS:0;828b3f090c91:35731 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-26T21:52:04,914 INFO [RS:1;828b3f090c91:39637 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-26T21:52:04,919 INFO [RS:0;828b3f090c91:35731 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-26T21:52:04,919 INFO [RS:1;828b3f090c91:39637 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-26T21:52:04,919 INFO [RS:1;828b3f090c91:39637 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-26T21:52:04,919 INFO [RS:0;828b3f090c91:35731 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-26T21:52:04,921 INFO [RS:0;828b3f090c91:35731 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-26T21:52:04,921 INFO [RS:1;828b3f090c91:39637 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-26T21:52:04,926 INFO [RS:1;828b3f090c91:39637 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-26T21:52:04,926 INFO [RS:0;828b3f090c91:35731 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-26T21:52:04,928 INFO [RS:1;828b3f090c91:39637 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-26T21:52:04,928 INFO [RS:0;828b3f090c91:35731 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-26T21:52:04,928 DEBUG [RS:1;828b3f090c91:39637 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/828b3f090c91:0, corePoolSize=1, maxPoolSize=1 2024-11-26T21:52:04,928 DEBUG [RS:0;828b3f090c91:35731 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/828b3f090c91:0, corePoolSize=1, maxPoolSize=1 2024-11-26T21:52:04,929 DEBUG [RS:1;828b3f090c91:39637 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/828b3f090c91:0, corePoolSize=1, maxPoolSize=1 2024-11-26T21:52:04,929 DEBUG [RS:0;828b3f090c91:35731 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/828b3f090c91:0, corePoolSize=1, maxPoolSize=1 2024-11-26T21:52:04,929 DEBUG [RS:1;828b3f090c91:39637 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/828b3f090c91:0, corePoolSize=1, maxPoolSize=1 2024-11-26T21:52:04,929 DEBUG [RS:0;828b3f090c91:35731 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/828b3f090c91:0, corePoolSize=1, maxPoolSize=1 2024-11-26T21:52:04,929 DEBUG [RS:0;828b3f090c91:35731 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/828b3f090c91:0, corePoolSize=1, maxPoolSize=1 2024-11-26T21:52:04,929 DEBUG [RS:1;828b3f090c91:39637 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/828b3f090c91:0, corePoolSize=1, maxPoolSize=1 2024-11-26T21:52:04,929 DEBUG [RS:0;828b3f090c91:35731 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/828b3f090c91:0, corePoolSize=1, maxPoolSize=1 2024-11-26T21:52:04,929 DEBUG [RS:1;828b3f090c91:39637 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/828b3f090c91:0, corePoolSize=1, maxPoolSize=1 2024-11-26T21:52:04,929 DEBUG [RS:0;828b3f090c91:35731 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/828b3f090c91:0, corePoolSize=2, maxPoolSize=2 2024-11-26T21:52:04,930 DEBUG [RS:1;828b3f090c91:39637 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/828b3f090c91:0, corePoolSize=2, maxPoolSize=2 2024-11-26T21:52:04,930 DEBUG [RS:0;828b3f090c91:35731 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/828b3f090c91:0, corePoolSize=1, maxPoolSize=1 2024-11-26T21:52:04,930 DEBUG [RS:1;828b3f090c91:39637 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/828b3f090c91:0, corePoolSize=1, maxPoolSize=1 2024-11-26T21:52:04,930 DEBUG [RS:0;828b3f090c91:35731 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/828b3f090c91:0, corePoolSize=1, maxPoolSize=1 2024-11-26T21:52:04,930 DEBUG [RS:1;828b3f090c91:39637 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/828b3f090c91:0, corePoolSize=1, maxPoolSize=1 2024-11-26T21:52:04,930 DEBUG [RS:0;828b3f090c91:35731 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/828b3f090c91:0, corePoolSize=1, maxPoolSize=1 2024-11-26T21:52:04,930 DEBUG [RS:1;828b3f090c91:39637 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/828b3f090c91:0, corePoolSize=1, maxPoolSize=1 2024-11-26T21:52:04,930 DEBUG [RS:0;828b3f090c91:35731 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/828b3f090c91:0, corePoolSize=1, maxPoolSize=1 2024-11-26T21:52:04,930 DEBUG [RS:1;828b3f090c91:39637 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/828b3f090c91:0, corePoolSize=1, maxPoolSize=1 2024-11-26T21:52:04,930 DEBUG [RS:0;828b3f090c91:35731 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/828b3f090c91:0, corePoolSize=1, maxPoolSize=1 2024-11-26T21:52:04,930 DEBUG [RS:1;828b3f090c91:39637 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/828b3f090c91:0, corePoolSize=1, maxPoolSize=1 2024-11-26T21:52:04,931 DEBUG [RS:0;828b3f090c91:35731 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/828b3f090c91:0, corePoolSize=1, maxPoolSize=1 2024-11-26T21:52:04,931 DEBUG [RS:1;828b3f090c91:39637 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/828b3f090c91:0, corePoolSize=1, maxPoolSize=1 2024-11-26T21:52:04,931 DEBUG [RS:0;828b3f090c91:35731 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/828b3f090c91:0, corePoolSize=3, maxPoolSize=3 2024-11-26T21:52:04,931 DEBUG [RS:1;828b3f090c91:39637 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/828b3f090c91:0, corePoolSize=3, maxPoolSize=3 2024-11-26T21:52:04,931 DEBUG [RS:0;828b3f090c91:35731 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/828b3f090c91:0, corePoolSize=3, maxPoolSize=3 2024-11-26T21:52:04,931 DEBUG [RS:1;828b3f090c91:39637 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/828b3f090c91:0, corePoolSize=3, maxPoolSize=3 2024-11-26T21:52:04,933 INFO [RS:0;828b3f090c91:35731 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-26T21:52:04,933 INFO [RS:1;828b3f090c91:39637 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-26T21:52:04,933 INFO [RS:0;828b3f090c91:35731 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-26T21:52:04,933 INFO [RS:1;828b3f090c91:39637 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-26T21:52:04,933 INFO [RS:0;828b3f090c91:35731 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-26T21:52:04,933 INFO [RS:1;828b3f090c91:39637 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-26T21:52:04,933 INFO [RS:0;828b3f090c91:35731 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-26T21:52:04,933 INFO [RS:1;828b3f090c91:39637 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-26T21:52:04,934 INFO [RS:0;828b3f090c91:35731 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-26T21:52:04,934 INFO [RS:1;828b3f090c91:39637 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-26T21:52:04,934 INFO [RS:0;828b3f090c91:35731 {}] hbase.ChoreService(168): Chore ScheduledChore name=828b3f090c91,35731,1732657923709-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-26T21:52:04,934 INFO [RS:1;828b3f090c91:39637 {}] hbase.ChoreService(168): Chore ScheduledChore name=828b3f090c91,39637,1732657923848-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-26T21:52:04,958 INFO [RS:0;828b3f090c91:35731 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-26T21:52:04,958 INFO [RS:1;828b3f090c91:39637 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-26T21:52:04,960 INFO [RS:0;828b3f090c91:35731 {}] hbase.ChoreService(168): Chore ScheduledChore name=828b3f090c91,35731,1732657923709-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-26T21:52:04,960 INFO [RS:1;828b3f090c91:39637 {}] hbase.ChoreService(168): Chore ScheduledChore name=828b3f090c91,39637,1732657923848-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-26T21:52:04,960 INFO [RS:1;828b3f090c91:39637 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-26T21:52:04,960 INFO [RS:0;828b3f090c91:35731 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-26T21:52:04,960 INFO [RS:0;828b3f090c91:35731 {}] regionserver.Replication(171): 828b3f090c91,35731,1732657923709 started 2024-11-26T21:52:04,960 INFO [RS:1;828b3f090c91:39637 {}] regionserver.Replication(171): 828b3f090c91,39637,1732657923848 started 2024-11-26T21:52:04,982 INFO [RS:0;828b3f090c91:35731 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-26T21:52:04,982 INFO [RS:0;828b3f090c91:35731 {}] regionserver.HRegionServer(1482): Serving as 828b3f090c91,35731,1732657923709, RpcServer on 828b3f090c91/172.17.0.2:35731, sessionid=0x1007e051f2e0001 2024-11-26T21:52:04,983 DEBUG [RS:0;828b3f090c91:35731 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-26T21:52:04,983 DEBUG [RS:0;828b3f090c91:35731 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 828b3f090c91,35731,1732657923709 2024-11-26T21:52:04,983 DEBUG [RS:0;828b3f090c91:35731 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '828b3f090c91,35731,1732657923709' 2024-11-26T21:52:04,984 DEBUG [RS:0;828b3f090c91:35731 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-26T21:52:04,985 DEBUG [RS:0;828b3f090c91:35731 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-26T21:52:04,985 DEBUG [RS:0;828b3f090c91:35731 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-26T21:52:04,985 DEBUG [RS:0;828b3f090c91:35731 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-26T21:52:04,986 DEBUG [RS:0;828b3f090c91:35731 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 828b3f090c91,35731,1732657923709 2024-11-26T21:52:04,986 DEBUG [RS:0;828b3f090c91:35731 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '828b3f090c91,35731,1732657923709' 2024-11-26T21:52:04,986 DEBUG [RS:0;828b3f090c91:35731 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-26T21:52:04,986 INFO [RS:1;828b3f090c91:39637 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-26T21:52:04,986 INFO [RS:1;828b3f090c91:39637 {}] regionserver.HRegionServer(1482): Serving as 828b3f090c91,39637,1732657923848, RpcServer on 828b3f090c91/172.17.0.2:39637, sessionid=0x1007e051f2e0002 2024-11-26T21:52:04,987 DEBUG [RS:0;828b3f090c91:35731 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-26T21:52:04,987 DEBUG [RS:1;828b3f090c91:39637 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-26T21:52:04,987 DEBUG [RS:1;828b3f090c91:39637 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 828b3f090c91,39637,1732657923848 2024-11-26T21:52:04,987 DEBUG [RS:1;828b3f090c91:39637 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '828b3f090c91,39637,1732657923848' 2024-11-26T21:52:04,987 DEBUG [RS:1;828b3f090c91:39637 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-26T21:52:04,987 DEBUG [RS:0;828b3f090c91:35731 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-26T21:52:04,987 INFO [RS:0;828b3f090c91:35731 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-26T21:52:04,987 INFO [RS:0;828b3f090c91:35731 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-26T21:52:04,988 DEBUG [RS:1;828b3f090c91:39637 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-26T21:52:04,988 DEBUG [RS:1;828b3f090c91:39637 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-26T21:52:04,988 DEBUG [RS:1;828b3f090c91:39637 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-26T21:52:04,988 DEBUG [RS:1;828b3f090c91:39637 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 828b3f090c91,39637,1732657923848 2024-11-26T21:52:04,988 DEBUG [RS:1;828b3f090c91:39637 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '828b3f090c91,39637,1732657923848' 2024-11-26T21:52:04,988 DEBUG [RS:1;828b3f090c91:39637 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-26T21:52:04,989 DEBUG [RS:1;828b3f090c91:39637 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-26T21:52:04,990 DEBUG [RS:1;828b3f090c91:39637 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-26T21:52:04,990 INFO [RS:1;828b3f090c91:39637 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-26T21:52:04,990 INFO [RS:1;828b3f090c91:39637 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-26T21:52:05,018 WARN [828b3f090c91:36093 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-26T21:52:05,093 INFO [RS:0;828b3f090c91:35731 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-26T21:52:05,093 INFO [RS:1;828b3f090c91:39637 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-26T21:52:05,094 INFO [RS:0;828b3f090c91:35731 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=828b3f090c91%2C35731%2C1732657923709, suffix=, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7430f913-2694-54cf-261c-654b69c5fe06/de866367-466a-3788-a91b-60c9e5aa6ab2/WALs/828b3f090c91,35731,1732657923709, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7430f913-2694-54cf-261c-654b69c5fe06/de866367-466a-3788-a91b-60c9e5aa6ab2/oldWALs, maxLogs=55 2024-11-26T21:52:05,094 INFO [RS:1;828b3f090c91:39637 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=828b3f090c91%2C39637%2C1732657923848, suffix=, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7430f913-2694-54cf-261c-654b69c5fe06/de866367-466a-3788-a91b-60c9e5aa6ab2/WALs/828b3f090c91,39637,1732657923848, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7430f913-2694-54cf-261c-654b69c5fe06/de866367-466a-3788-a91b-60c9e5aa6ab2/oldWALs, maxLogs=55 2024-11-26T21:52:05,116 INFO [RS:0;828b3f090c91:35731 {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7430f913-2694-54cf-261c-654b69c5fe06/de866367-466a-3788-a91b-60c9e5aa6ab2/WALs/828b3f090c91,35731,1732657923709/828b3f090c91%2C35731%2C1732657923709.1732657925097 2024-11-26T21:52:05,116 DEBUG [RS:0;828b3f090c91:35731 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-11-26T21:52:05,118 INFO [RS:1;828b3f090c91:39637 {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7430f913-2694-54cf-261c-654b69c5fe06/de866367-466a-3788-a91b-60c9e5aa6ab2/WALs/828b3f090c91,39637,1732657923848/828b3f090c91%2C39637%2C1732657923848.1732657925097 2024-11-26T21:52:05,118 DEBUG [RS:1;828b3f090c91:39637 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-11-26T21:52:05,272 DEBUG [828b3f090c91:36093 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=2, allServersCount=2 2024-11-26T21:52:05,287 DEBUG [828b3f090c91:36093 {}] balancer.BalancerClusterState(204): Hosts are {828b3f090c91=0} racks are {/default-rack=0} 2024-11-26T21:52:05,290 DEBUG [828b3f090c91:36093 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-11-26T21:52:05,291 DEBUG [828b3f090c91:36093 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-11-26T21:52:05,291 DEBUG [828b3f090c91:36093 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-11-26T21:52:05,291 DEBUG [828b3f090c91:36093 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-11-26T21:52:05,291 INFO [828b3f090c91:36093 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-11-26T21:52:05,291 INFO [828b3f090c91:36093 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-11-26T21:52:05,291 DEBUG [828b3f090c91:36093 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-11-26T21:52:05,299 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=828b3f090c91,35731,1732657923709 2024-11-26T21:52:05,303 INFO [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 828b3f090c91,35731,1732657923709, state=OPENING 2024-11-26T21:52:05,307 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2024-11-26T21:52:05,309 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39637-0x1007e051f2e0002, quorum=127.0.0.1:64198, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-26T21:52:05,309 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36093-0x1007e051f2e0000, quorum=127.0.0.1:64198, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-26T21:52:05,309 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35731-0x1007e051f2e0001, quorum=127.0.0.1:64198, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-26T21:52:05,310 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-26T21:52:05,310 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-26T21:52:05,310 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-26T21:52:05,311 DEBUG [PEWorker-3 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-26T21:52:05,314 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=828b3f090c91,35731,1732657923709}] 2024-11-26T21:52:05,483 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-26T21:52:05,486 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50069, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-26T21:52:05,494 INFO [RS_OPEN_META-regionserver/828b3f090c91:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-11-26T21:52:05,494 INFO [RS_OPEN_META-regionserver/828b3f090c91:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-26T21:52:05,495 INFO [RS_OPEN_META-regionserver/828b3f090c91:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-11-26T21:52:05,496 INFO [RS_OPEN_META-regionserver/828b3f090c91:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=828b3f090c91%2C35731%2C1732657923709.meta, suffix=.meta, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7430f913-2694-54cf-261c-654b69c5fe06/de866367-466a-3788-a91b-60c9e5aa6ab2/WALs/828b3f090c91,35731,1732657923709, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7430f913-2694-54cf-261c-654b69c5fe06/de866367-466a-3788-a91b-60c9e5aa6ab2/oldWALs, maxLogs=55 2024-11-26T21:52:05,522 INFO [RS_OPEN_META-regionserver/828b3f090c91:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7430f913-2694-54cf-261c-654b69c5fe06/de866367-466a-3788-a91b-60c9e5aa6ab2/WALs/828b3f090c91,35731,1732657923709/828b3f090c91%2C35731%2C1732657923709.meta.1732657925499.meta 2024-11-26T21:52:05,523 DEBUG [RS_OPEN_META-regionserver/828b3f090c91:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-11-26T21:52:05,523 DEBUG [RS_OPEN_META-regionserver/828b3f090c91:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-11-26T21:52:05,526 DEBUG [RS_OPEN_META-regionserver/828b3f090c91:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-11-26T21:52:05,529 DEBUG [RS_OPEN_META-regionserver/828b3f090c91:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-11-26T21:52:05,534 INFO [RS_OPEN_META-regionserver/828b3f090c91:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-11-26T21:52:05,538 DEBUG [RS_OPEN_META-regionserver/828b3f090c91:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-11-26T21:52:05,538 DEBUG [RS_OPEN_META-regionserver/828b3f090c91:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-26T21:52:05,539 DEBUG [RS_OPEN_META-regionserver/828b3f090c91:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-11-26T21:52:05,539 DEBUG [RS_OPEN_META-regionserver/828b3f090c91:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-11-26T21:52:05,540 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family info of region 1588230740 2024-11-26T21:52:05,541 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-26T21:52:05,541 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T21:52:05,542 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-26T21:52:05,542 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family ns of region 1588230740 2024-11-26T21:52:05,543 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-26T21:52:05,543 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T21:52:05,543 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-26T21:52:05,543 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family rep_barrier of region 1588230740 2024-11-26T21:52:05,544 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-26T21:52:05,544 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T21:52:05,544 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-26T21:52:05,544 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family table of region 1588230740 2024-11-26T21:52:05,545 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-26T21:52:05,545 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T21:52:05,546 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-26T21:52:05,546 DEBUG [RS_OPEN_META-regionserver/828b3f090c91:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-26T21:52:05,547 DEBUG [RS_OPEN_META-regionserver/828b3f090c91:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7430f913-2694-54cf-261c-654b69c5fe06/de866367-466a-3788-a91b-60c9e5aa6ab2/data/hbase/meta/1588230740 2024-11-26T21:52:05,548 DEBUG [RS_OPEN_META-regionserver/828b3f090c91:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7430f913-2694-54cf-261c-654b69c5fe06/de866367-466a-3788-a91b-60c9e5aa6ab2/data/hbase/meta/1588230740 2024-11-26T21:52:05,548 DEBUG [RS_OPEN_META-regionserver/828b3f090c91:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-26T21:52:05,548 DEBUG [RS_OPEN_META-regionserver/828b3f090c91:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-26T21:52:05,549 DEBUG [RS_OPEN_META-regionserver/828b3f090c91:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-26T21:52:05,550 DEBUG [RS_OPEN_META-regionserver/828b3f090c91:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-26T21:52:05,550 INFO [RS_OPEN_META-regionserver/828b3f090c91:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=74412485, jitterRate=0.10883243381977081}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-26T21:52:05,551 DEBUG [RS_OPEN_META-regionserver/828b3f090c91:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-11-26T21:52:05,552 DEBUG [RS_OPEN_META-regionserver/828b3f090c91:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1732657925539Writing region info on filesystem at 1732657925540 (+1 ms)Initializing all the Stores at 1732657925540Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732657925540Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732657925540Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732657925540Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732657925540Cleaning up temporary data from old regions at 1732657925548 (+8 ms)Running coprocessor post-open hooks at 1732657925551 (+3 ms)Region opened successfully at 1732657925552 (+1 ms) 2024-11-26T21:52:05,558 INFO [RS_OPEN_META-regionserver/828b3f090c91:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1732657925476 2024-11-26T21:52:05,567 DEBUG [RS_OPEN_META-regionserver/828b3f090c91:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-11-26T21:52:05,567 INFO [RS_OPEN_META-regionserver/828b3f090c91:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-11-26T21:52:05,568 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=828b3f090c91,35731,1732657923709 2024-11-26T21:52:05,570 INFO [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 828b3f090c91,35731,1732657923709, state=OPEN 2024-11-26T21:52:05,572 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35731-0x1007e051f2e0001, quorum=127.0.0.1:64198, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-26T21:52:05,572 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39637-0x1007e051f2e0002, quorum=127.0.0.1:64198, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-26T21:52:05,572 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36093-0x1007e051f2e0000, quorum=127.0.0.1:64198, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-26T21:52:05,572 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-26T21:52:05,572 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-26T21:52:05,572 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-26T21:52:05,572 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=828b3f090c91,35731,1732657923709 2024-11-26T21:52:05,575 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2 2024-11-26T21:52:05,576 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=828b3f090c91,35731,1732657923709 in 260 msec 2024-11-26T21:52:05,581 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1 2024-11-26T21:52:05,581 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 722 msec 2024-11-26T21:52:05,582 DEBUG [PEWorker-2 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-26T21:52:05,582 INFO [PEWorker-2 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces 2024-11-26T21:52:05,601 DEBUG [PEWorker-2 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-26T21:52:05,602 DEBUG [PEWorker-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=828b3f090c91,35731,1732657923709, seqNum=-1] 2024-11-26T21:52:05,619 DEBUG [PEWorker-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-26T21:52:05,621 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:32941, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-26T21:52:05,639 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 984 msec 2024-11-26T21:52:05,639 INFO [master/828b3f090c91:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1732657925639, completionTime=-1 2024-11-26T21:52:05,642 INFO [master/828b3f090c91:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=2; waited=0ms, expected min=2 server(s), max=2 server(s), master is running 2024-11-26T21:52:05,642 DEBUG [master/828b3f090c91:0:becomeActiveMaster {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-26T21:52:05,665 INFO [master/828b3f090c91:0:becomeActiveMaster {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-11-26T21:52:05,665 INFO [master/828b3f090c91:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732657985665 2024-11-26T21:52:05,665 INFO [master/828b3f090c91:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732658045665 2024-11-26T21:52:05,665 INFO [master/828b3f090c91:0:becomeActiveMaster {}] assignment.AssignmentManager(1783): Joined the cluster in 23 msec 2024-11-26T21:52:05,667 DEBUG [master/828b3f090c91:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 1588230740 changed from -1.0 to 0.0, refreshing cache 2024-11-26T21:52:05,673 INFO [master/828b3f090c91:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=828b3f090c91,36093,1732657923024-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-26T21:52:05,673 INFO [master/828b3f090c91:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=828b3f090c91,36093,1732657923024-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-26T21:52:05,673 INFO [master/828b3f090c91:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=828b3f090c91,36093,1732657923024-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-26T21:52:05,674 INFO [master/828b3f090c91:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-828b3f090c91:36093, period=300000, unit=MILLISECONDS is enabled. 2024-11-26T21:52:05,675 INFO [master/828b3f090c91:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2024-11-26T21:52:05,675 INFO [master/828b3f090c91:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled. 2024-11-26T21:52:05,682 DEBUG [master/828b3f090c91:0.Chore.1 {}] janitor.CatalogJanitor(180): 2024-11-26T21:52:05,701 INFO [master/828b3f090c91:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 1.742sec 2024-11-26T21:52:05,702 INFO [master/828b3f090c91:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2024-11-26T21:52:05,703 INFO [master/828b3f090c91:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2024-11-26T21:52:05,704 INFO [master/828b3f090c91:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2024-11-26T21:52:05,704 INFO [master/828b3f090c91:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2024-11-26T21:52:05,704 INFO [master/828b3f090c91:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2024-11-26T21:52:05,705 INFO [master/828b3f090c91:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=828b3f090c91,36093,1732657923024-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-26T21:52:05,705 INFO [master/828b3f090c91:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=828b3f090c91,36093,1732657923024-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2024-11-26T21:52:05,706 DEBUG [master/828b3f090c91:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds 2024-11-26T21:52:05,707 INFO [master/828b3f090c91:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2024-11-26T21:52:05,708 INFO [master/828b3f090c91:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=828b3f090c91,36093,1732657923024-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-26T21:52:05,800 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4248d59d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T21:52:05,802 DEBUG [Time-limited test {}] nio.NioEventLoop(110): -Dio.netty.noKeySetOptimization: false 2024-11-26T21:52:05,803 DEBUG [Time-limited test {}] nio.NioEventLoop(111): -Dio.netty.selectorAutoRebuildThreshold: 512 2024-11-26T21:52:05,807 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 828b3f090c91,36093,-1 for getting cluster id 2024-11-26T21:52:05,810 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-26T21:52:05,818 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'a0458ab8-5c44-4513-aa4c-756a7f2c1a4b' 2024-11-26T21:52:05,820 DEBUG [RPCClient-NioEventLoopGroup-5-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-26T21:52:05,820 DEBUG [RPCClient-NioEventLoopGroup-5-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "a0458ab8-5c44-4513-aa4c-756a7f2c1a4b" 2024-11-26T21:52:05,822 DEBUG [RPCClient-NioEventLoopGroup-5-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@15372055, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T21:52:05,822 DEBUG [RPCClient-NioEventLoopGroup-5-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [828b3f090c91,36093,-1] 2024-11-26T21:52:05,824 DEBUG [RPCClient-NioEventLoopGroup-5-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-26T21:52:05,826 DEBUG [RPCClient-NioEventLoopGroup-5-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T21:52:05,827 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36782, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-26T21:52:05,830 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5f82a59d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T21:52:05,831 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-26T21:52:05,837 DEBUG [RPCClient-NioEventLoopGroup-5-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=828b3f090c91,35731,1732657923709, seqNum=-1] 2024-11-26T21:52:05,838 DEBUG [RPCClient-NioEventLoopGroup-5-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-26T21:52:05,841 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:55614, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-26T21:52:05,876 INFO [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=828b3f090c91,36093,1732657923024 2024-11-26T21:52:05,881 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-26T21:52:05,888 DEBUG [RPCClient-NioEventLoopGroup-5-2 {}] client.AsyncConnectionImpl(321): The fetched master address is 828b3f090c91,36093,1732657923024 2024-11-26T21:52:05,891 DEBUG [RPCClient-NioEventLoopGroup-5-2 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@538d74c4 2024-11-26T21:52:05,893 DEBUG [RPCClient-NioEventLoopGroup-5-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-26T21:52:05,898 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36798, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-26T21:52:05,904 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36093 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'testBlockEvictionOnRegionMove1732657925876', {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-26T21:52:05,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36093 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=testBlockEvictionOnRegionMove1732657925876 2024-11-26T21:52:05,914 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=testBlockEvictionOnRegionMove1732657925876 execute state=CREATE_TABLE_PRE_OPERATION 2024-11-26T21:52:05,916 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36093 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "testBlockEvictionOnRegionMove1732657925876" procId is: 4 2024-11-26T21:52:05,916 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T21:52:05,918 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=testBlockEvictionOnRegionMove1732657925876 execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-26T21:52:05,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36093 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-26T21:52:05,930 INFO [RegionOpenAndInit-testBlockEvictionOnRegionMove1732657925876-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 1df223c3fe5e10053cc5cd1832f3c22e, NAME => 'testBlockEvictionOnRegionMove1732657925876,,1732657925900.1df223c3fe5e10053cc5cd1832f3c22e.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testBlockEvictionOnRegionMove1732657925876', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7430f913-2694-54cf-261c-654b69c5fe06/de866367-466a-3788-a91b-60c9e5aa6ab2 2024-11-26T21:52:05,936 DEBUG [RegionOpenAndInit-testBlockEvictionOnRegionMove1732657925876-pool-0 {}] regionserver.HRegion(898): Instantiated testBlockEvictionOnRegionMove1732657925876,,1732657925900.1df223c3fe5e10053cc5cd1832f3c22e.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-26T21:52:05,936 DEBUG [RegionOpenAndInit-testBlockEvictionOnRegionMove1732657925876-pool-0 {}] regionserver.HRegion(1722): Closing 1df223c3fe5e10053cc5cd1832f3c22e, disabling compactions & flushes 2024-11-26T21:52:05,936 INFO [RegionOpenAndInit-testBlockEvictionOnRegionMove1732657925876-pool-0 {}] regionserver.HRegion(1755): Closing region testBlockEvictionOnRegionMove1732657925876,,1732657925900.1df223c3fe5e10053cc5cd1832f3c22e. 2024-11-26T21:52:05,936 DEBUG [RegionOpenAndInit-testBlockEvictionOnRegionMove1732657925876-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on testBlockEvictionOnRegionMove1732657925876,,1732657925900.1df223c3fe5e10053cc5cd1832f3c22e. 2024-11-26T21:52:05,936 DEBUG [RegionOpenAndInit-testBlockEvictionOnRegionMove1732657925876-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on testBlockEvictionOnRegionMove1732657925876,,1732657925900.1df223c3fe5e10053cc5cd1832f3c22e. after waiting 0 ms 2024-11-26T21:52:05,937 DEBUG [RegionOpenAndInit-testBlockEvictionOnRegionMove1732657925876-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region testBlockEvictionOnRegionMove1732657925876,,1732657925900.1df223c3fe5e10053cc5cd1832f3c22e. 2024-11-26T21:52:05,937 INFO [RegionOpenAndInit-testBlockEvictionOnRegionMove1732657925876-pool-0 {}] regionserver.HRegion(1973): Closed testBlockEvictionOnRegionMove1732657925876,,1732657925900.1df223c3fe5e10053cc5cd1832f3c22e. 2024-11-26T21:52:05,937 DEBUG [RegionOpenAndInit-testBlockEvictionOnRegionMove1732657925876-pool-0 {}] regionserver.HRegion(1676): Region close journal for 1df223c3fe5e10053cc5cd1832f3c22e: Waiting for close lock at 1732657925936Disabling compacts and flushes for region at 1732657925936Disabling writes for close at 1732657925937 (+1 ms)Writing region close event to WAL at 1732657925937Closed at 1732657925937 2024-11-26T21:52:05,938 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=testBlockEvictionOnRegionMove1732657925876 execute state=CREATE_TABLE_ADD_TO_META 2024-11-26T21:52:05,942 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testBlockEvictionOnRegionMove1732657925876,,1732657925900.1df223c3fe5e10053cc5cd1832f3c22e.","families":{"info":[{"qualifier":"regioninfo","vlen":76,"tag":[],"timestamp":"1732657925938"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732657925938"}]},"ts":"1732657925938"} 2024-11-26T21:52:05,946 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-26T21:52:05,948 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=testBlockEvictionOnRegionMove1732657925876 execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-26T21:52:05,950 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testBlockEvictionOnRegionMove1732657925876","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732657925948"}]},"ts":"1732657925948"} 2024-11-26T21:52:05,953 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=testBlockEvictionOnRegionMove1732657925876, state=ENABLING in hbase:meta 2024-11-26T21:52:05,954 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(204): Hosts are {828b3f090c91=0} racks are {/default-rack=0} 2024-11-26T21:52:05,955 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-11-26T21:52:05,955 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-11-26T21:52:05,955 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-11-26T21:52:05,955 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-11-26T21:52:05,955 INFO [PEWorker-3 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-11-26T21:52:05,955 INFO [PEWorker-3 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-11-26T21:52:05,955 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-11-26T21:52:05,956 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1732657925876, region=1df223c3fe5e10053cc5cd1832f3c22e, ASSIGN}] 2024-11-26T21:52:05,958 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1732657925876, region=1df223c3fe5e10053cc5cd1832f3c22e, ASSIGN 2024-11-26T21:52:05,959 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1732657925876, region=1df223c3fe5e10053cc5cd1832f3c22e, ASSIGN; state=OFFLINE, location=828b3f090c91,35731,1732657923709; forceNewPlan=false, retain=false 2024-11-26T21:52:06,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36093 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-26T21:52:06,116 INFO [828b3f090c91:36093 {}] balancer.BaseLoadBalancer(388): Reassigned 1 regions. 1 retained the pre-restart assignment. 2024-11-26T21:52:06,117 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=1df223c3fe5e10053cc5cd1832f3c22e, regionState=OPENING, regionLocation=828b3f090c91,35731,1732657923709 2024-11-26T21:52:06,124 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-4-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1732657925876, region=1df223c3fe5e10053cc5cd1832f3c22e, ASSIGN because future has completed 2024-11-26T21:52:06,125 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1df223c3fe5e10053cc5cd1832f3c22e, server=828b3f090c91,35731,1732657923709}] 2024-11-26T21:52:06,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36093 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-26T21:52:06,281 INFO [RS_OPEN_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(132): Open testBlockEvictionOnRegionMove1732657925876,,1732657925900.1df223c3fe5e10053cc5cd1832f3c22e. 2024-11-26T21:52:06,281 DEBUG [RS_OPEN_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7752): Opening region: {ENCODED => 1df223c3fe5e10053cc5cd1832f3c22e, NAME => 'testBlockEvictionOnRegionMove1732657925876,,1732657925900.1df223c3fe5e10053cc5cd1832f3c22e.', STARTKEY => '', ENDKEY => ''} 2024-11-26T21:52:06,282 DEBUG [RS_OPEN_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testBlockEvictionOnRegionMove1732657925876 1df223c3fe5e10053cc5cd1832f3c22e 2024-11-26T21:52:06,282 DEBUG [RS_OPEN_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(898): Instantiated testBlockEvictionOnRegionMove1732657925876,,1732657925900.1df223c3fe5e10053cc5cd1832f3c22e.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-26T21:52:06,282 DEBUG [RS_OPEN_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7794): checking encryption for 1df223c3fe5e10053cc5cd1832f3c22e 2024-11-26T21:52:06,282 DEBUG [RS_OPEN_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7797): checking classloading for 1df223c3fe5e10053cc5cd1832f3c22e 2024-11-26T21:52:06,283 INFO [StoreOpener-1df223c3fe5e10053cc5cd1832f3c22e-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family family of region 1df223c3fe5e10053cc5cd1832f3c22e 2024-11-26T21:52:06,288 INFO [StoreOpener-1df223c3fe5e10053cc5cd1832f3c22e-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1df223c3fe5e10053cc5cd1832f3c22e columnFamilyName family 2024-11-26T21:52:06,288 DEBUG [StoreOpener-1df223c3fe5e10053cc5cd1832f3c22e-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T21:52:06,288 INFO [StoreOpener-1df223c3fe5e10053cc5cd1832f3c22e-1 {}] regionserver.HStore(327): Store=1df223c3fe5e10053cc5cd1832f3c22e/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-26T21:52:06,289 DEBUG [RS_OPEN_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1038): replaying wal for 1df223c3fe5e10053cc5cd1832f3c22e 2024-11-26T21:52:06,290 DEBUG [RS_OPEN_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7430f913-2694-54cf-261c-654b69c5fe06/de866367-466a-3788-a91b-60c9e5aa6ab2/data/default/testBlockEvictionOnRegionMove1732657925876/1df223c3fe5e10053cc5cd1832f3c22e 2024-11-26T21:52:06,290 DEBUG [RS_OPEN_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7430f913-2694-54cf-261c-654b69c5fe06/de866367-466a-3788-a91b-60c9e5aa6ab2/data/default/testBlockEvictionOnRegionMove1732657925876/1df223c3fe5e10053cc5cd1832f3c22e 2024-11-26T21:52:06,291 DEBUG [RS_OPEN_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1048): stopping wal replay for 1df223c3fe5e10053cc5cd1832f3c22e 2024-11-26T21:52:06,291 DEBUG [RS_OPEN_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1060): Cleaning up temporary data for 1df223c3fe5e10053cc5cd1832f3c22e 2024-11-26T21:52:06,292 DEBUG [RS_OPEN_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1093): writing seq id for 1df223c3fe5e10053cc5cd1832f3c22e 2024-11-26T21:52:06,298 DEBUG [RS_OPEN_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7430f913-2694-54cf-261c-654b69c5fe06/de866367-466a-3788-a91b-60c9e5aa6ab2/data/default/testBlockEvictionOnRegionMove1732657925876/1df223c3fe5e10053cc5cd1832f3c22e/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-26T21:52:06,298 INFO [RS_OPEN_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1114): Opened 1df223c3fe5e10053cc5cd1832f3c22e; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=71363919, jitterRate=0.06340526044368744}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-26T21:52:06,298 DEBUG [RS_OPEN_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1df223c3fe5e10053cc5cd1832f3c22e 2024-11-26T21:52:06,299 DEBUG [RS_OPEN_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1006): Region open journal for 1df223c3fe5e10053cc5cd1832f3c22e: Running coprocessor pre-open hook at 1732657926283Writing region info on filesystem at 1732657926283Initializing all the Stores at 1732657926283Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732657926283Cleaning up temporary data from old regions at 1732657926291 (+8 ms)Running coprocessor post-open hooks at 1732657926298 (+7 ms)Region opened successfully at 1732657926299 (+1 ms) 2024-11-26T21:52:06,300 INFO [RS_OPEN_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2236): Post open deploy tasks for testBlockEvictionOnRegionMove1732657925876,,1732657925900.1df223c3fe5e10053cc5cd1832f3c22e., pid=6, masterSystemTime=1732657926278 2024-11-26T21:52:06,304 DEBUG [RS_OPEN_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2266): Finished post open deploy task for testBlockEvictionOnRegionMove1732657925876,,1732657925900.1df223c3fe5e10053cc5cd1832f3c22e. 2024-11-26T21:52:06,304 INFO [RS_OPEN_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(153): Opened testBlockEvictionOnRegionMove1732657925876,,1732657925900.1df223c3fe5e10053cc5cd1832f3c22e. 2024-11-26T21:52:06,304 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=1df223c3fe5e10053cc5cd1832f3c22e, regionState=OPEN, openSeqNum=2, regionLocation=828b3f090c91,35731,1732657923709 2024-11-26T21:52:06,309 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-4-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1df223c3fe5e10053cc5cd1832f3c22e, server=828b3f090c91,35731,1732657923709 because future has completed 2024-11-26T21:52:06,313 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=6, resume processing ppid=5 2024-11-26T21:52:06,314 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure 1df223c3fe5e10053cc5cd1832f3c22e, server=828b3f090c91,35731,1732657923709 in 186 msec 2024-11-26T21:52:06,316 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=5, resume processing ppid=4 2024-11-26T21:52:06,316 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1732657925876, region=1df223c3fe5e10053cc5cd1832f3c22e, ASSIGN in 358 msec 2024-11-26T21:52:06,317 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=testBlockEvictionOnRegionMove1732657925876 execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-26T21:52:06,317 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testBlockEvictionOnRegionMove1732657925876","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732657926317"}]},"ts":"1732657926317"} 2024-11-26T21:52:06,320 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=testBlockEvictionOnRegionMove1732657925876, state=ENABLED in hbase:meta 2024-11-26T21:52:06,321 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=testBlockEvictionOnRegionMove1732657925876 execute state=CREATE_TABLE_POST_OPERATION 2024-11-26T21:52:06,323 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=testBlockEvictionOnRegionMove1732657925876 in 414 msec 2024-11-26T21:52:06,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36093 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-26T21:52:06,558 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table testBlockEvictionOnRegionMove1732657925876 get assigned. Timeout = 60000ms 2024-11-26T21:52:06,558 INFO [RPCClient-NioEventLoopGroup-5-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:testBlockEvictionOnRegionMove1732657925876 completed 2024-11-26T21:52:06,564 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-26T21:52:06,571 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table testBlockEvictionOnRegionMove1732657925876 assigned to meta. Checking AM states. 2024-11-26T21:52:06,571 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-26T21:52:06,572 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table testBlockEvictionOnRegionMove1732657925876 assigned. 2024-11-26T21:52:06,581 DEBUG [RPCClient-NioEventLoopGroup-5-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testBlockEvictionOnRegionMove1732657925876', row='row1', locateType=CURRENT is [region=testBlockEvictionOnRegionMove1732657925876,,1732657925900.1df223c3fe5e10053cc5cd1832f3c22e., hostname=828b3f090c91,35731,1732657923709, seqNum=2] 2024-11-26T21:52:06,593 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 1df223c3fe5e10053cc5cd1832f3c22e 1/1 column families, dataSize=78 B heapSize=480 B 2024-11-26T21:52:06,618 DEBUG [Time-limited test {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 4d545b55745947f0a66d417912877126 2024-11-26T21:52:06,654 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7430f913-2694-54cf-261c-654b69c5fe06/de866367-466a-3788-a91b-60c9e5aa6ab2/data/default/testBlockEvictionOnRegionMove1732657925876/1df223c3fe5e10053cc5cd1832f3c22e/.tmp/family/4d545b55745947f0a66d417912877126 is 43, key is row1/family:qf1/1/Put/seqid=0 2024-11-26T21:52:06,658 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=78 B at sequenceid=6 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7430f913-2694-54cf-261c-654b69c5fe06/de866367-466a-3788-a91b-60c9e5aa6ab2/data/default/testBlockEvictionOnRegionMove1732657925876/1df223c3fe5e10053cc5cd1832f3c22e/.tmp/family/4d545b55745947f0a66d417912877126 2024-11-26T21:52:06,693 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7430f913-2694-54cf-261c-654b69c5fe06/de866367-466a-3788-a91b-60c9e5aa6ab2/data/default/testBlockEvictionOnRegionMove1732657925876/1df223c3fe5e10053cc5cd1832f3c22e/.tmp/family/4d545b55745947f0a66d417912877126 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7430f913-2694-54cf-261c-654b69c5fe06/de866367-466a-3788-a91b-60c9e5aa6ab2/data/default/testBlockEvictionOnRegionMove1732657925876/1df223c3fe5e10053cc5cd1832f3c22e/family/4d545b55745947f0a66d417912877126 2024-11-26T21:52:06,695 DEBUG [Time-limited test {}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7430f913-2694-54cf-261c-654b69c5fe06/de866367-466a-3788-a91b-60c9e5aa6ab2/data/default/testBlockEvictionOnRegionMove1732657925876/1df223c3fe5e10053cc5cd1832f3c22e/family/4d545b55745947f0a66d417912877126, delay=964 ms 2024-11-26T21:52:06,698 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7430f913-2694-54cf-261c-654b69c5fe06/de866367-466a-3788-a91b-60c9e5aa6ab2/data/default/testBlockEvictionOnRegionMove1732657925876/1df223c3fe5e10053cc5cd1832f3c22e/family/4d545b55745947f0a66d417912877126, entries=2, sequenceid=6, filesize=4.9 K 2024-11-26T21:52:06,706 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~78 B/78, heapSize ~464 B/464, currentSize=0 B/0 for 1df223c3fe5e10053cc5cd1832f3c22e in 111ms, sequenceid=6, compaction requested=false 2024-11-26T21:52:06,708 DEBUG [Time-limited test {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'testBlockEvictionOnRegionMove1732657925876' 2024-11-26T21:52:06,711 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for 1df223c3fe5e10053cc5cd1832f3c22e: 2024-11-26T21:52:07,662 DEBUG [hfile-prefetch-1732657926695 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 4d545b55745947f0a66d417912877126_0. 2024-11-26T21:52:07,669 DEBUG [hfile-prefetch-1732657926695 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 4d545b55745947f0a66d417912877126_117. 2024-11-26T21:52:07,669 DEBUG [hfile-prefetch-1732657926695 {}] bucket.BucketCache(2281): Notifying caching completed for file file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7430f913-2694-54cf-261c-654b69c5fe06/de866367-466a-3788-a91b-60c9e5aa6ab2/data/default/testBlockEvictionOnRegionMove1732657925876/1df223c3fe5e10053cc5cd1832f3c22e/family/4d545b55745947f0a66d417912877126, with total blocks 2, and data blocks 1 2024-11-26T21:52:07,669 DEBUG [hfile-prefetch-1732657926695 {}] bucket.BucketCache(2285): iterating over 0 entries in the backing map 2024-11-26T21:52:07,670 DEBUG [hfile-prefetch-1732657926695 {}] bucket.BucketCache(2308): Prefetch executor completed for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7430f913-2694-54cf-261c-654b69c5fe06/de866367-466a-3788-a91b-60c9e5aa6ab2/data/default/testBlockEvictionOnRegionMove1732657925876/1df223c3fe5e10053cc5cd1832f3c22e/family/4d545b55745947f0a66d417912877126, but only 0 data blocks were cached. Total data blocks for file: 1. Checking for blocks pending cache in cache writer queue. 2024-11-26T21:52:07,670 DEBUG [hfile-prefetch-1732657926695 {}] bucket.BucketCache(2318): There are still blocks pending caching for file file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7430f913-2694-54cf-261c-654b69c5fe06/de866367-466a-3788-a91b-60c9e5aa6ab2/data/default/testBlockEvictionOnRegionMove1732657925876/1df223c3fe5e10053cc5cd1832f3c22e/family/4d545b55745947f0a66d417912877126. Will sleep 100ms and try the verification again. 2024-11-26T21:52:07,675 DEBUG [bucket-cache-persister {}] bucket.BucketCachePersister(47): Cache is inconsistent, persisting to disk 2024-11-26T21:52:07,676 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1487): Thread bucket-cache-persister started persisting bucket cache to file 2024-11-26T21:52:07,676 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1494): Persist in new chunked persistence format. 2024-11-26T21:52:07,676 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1673): persistToFile: before persisting backing map size: 0, fullycachedFiles size: 0, chunkSize: 10000 2024-11-26T21:52:07,690 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1680): persistToFile: after persisting backing map size: 0, fullycachedFiles size: 0 2024-11-26T21:52:07,691 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1498): PersistToFile: after persisting backing map size: 0, fullycachedFiles size: 0, file name: bucket.persistence1732657927676 2024-11-26T21:52:07,691 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1509): Thread bucket-cache-persister finished persisting bucket cache to file, renaming 2024-11-26T21:52:07,726 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36093 {}] master.HMaster(2410): Client=jenkins//172.17.0.2 move hri=1df223c3fe5e10053cc5cd1832f3c22e, source=828b3f090c91,35731,1732657923709, destination=828b3f090c91,39637,1732657923848, warming up region on 828b3f090c91,39637,1732657923848 2024-11-26T21:52:07,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36093 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-26T21:52:07,728 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36093 {}] master.HMaster(2414): Client=jenkins//172.17.0.2 move hri=1df223c3fe5e10053cc5cd1832f3c22e, source=828b3f090c91,35731,1732657923709, destination=828b3f090c91,39637,1732657923848, running balancer 2024-11-26T21:52:07,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36093 {}] procedure2.ProcedureExecutor(1139): Stored pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1732657925876, region=1df223c3fe5e10053cc5cd1832f3c22e, REOPEN/MOVE 2024-11-26T21:52:07,730 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-4-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54497, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-26T21:52:07,730 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1732657925876, region=1df223c3fe5e10053cc5cd1832f3c22e, REOPEN/MOVE 2024-11-26T21:52:07,731 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39637 {}] regionserver.RSRpcServices(2066): Warmup testBlockEvictionOnRegionMove1732657925876,,1732657925900.1df223c3fe5e10053cc5cd1832f3c22e. 2024-11-26T21:52:07,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39637 {}] regionserver.HRegion(7855): Warmup {ENCODED => 1df223c3fe5e10053cc5cd1832f3c22e, NAME => 'testBlockEvictionOnRegionMove1732657925876,,1732657925900.1df223c3fe5e10053cc5cd1832f3c22e.', STARTKEY => '', ENDKEY => ''} 2024-11-26T21:52:07,731 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=1df223c3fe5e10053cc5cd1832f3c22e, regionState=CLOSING, regionLocation=828b3f090c91,35731,1732657923709 2024-11-26T21:52:07,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39637 {}] regionserver.HRegion(898): Instantiated testBlockEvictionOnRegionMove1732657925876,,1732657925900.1df223c3fe5e10053cc5cd1832f3c22e.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-26T21:52:07,732 INFO [StoreOpener-1df223c3fe5e10053cc5cd1832f3c22e-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family family of region 1df223c3fe5e10053cc5cd1832f3c22e 2024-11-26T21:52:07,733 INFO [StoreOpener-1df223c3fe5e10053cc5cd1832f3c22e-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1df223c3fe5e10053cc5cd1832f3c22e columnFamilyName family 2024-11-26T21:52:07,733 DEBUG [StoreOpener-1df223c3fe5e10053cc5cd1832f3c22e-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T21:52:07,735 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-4-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1732657925876, region=1df223c3fe5e10053cc5cd1832f3c22e, REOPEN/MOVE because future has completed 2024-11-26T21:52:07,735 DEBUG [PEWorker-3 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: true 2024-11-26T21:52:07,736 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=8, ppid=7, state=RUNNABLE, hasLock=false; CloseRegionProcedure 1df223c3fe5e10053cc5cd1832f3c22e, server=828b3f090c91,35731,1732657923709}] 2024-11-26T21:52:07,737 DEBUG [StoreOpener-1df223c3fe5e10053cc5cd1832f3c22e-1 {}] regionserver.StoreEngine(278): loaded file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7430f913-2694-54cf-261c-654b69c5fe06/de866367-466a-3788-a91b-60c9e5aa6ab2/data/default/testBlockEvictionOnRegionMove1732657925876/1df223c3fe5e10053cc5cd1832f3c22e/family/4d545b55745947f0a66d417912877126 2024-11-26T21:52:07,737 INFO [StoreOpener-1df223c3fe5e10053cc5cd1832f3c22e-1 {}] regionserver.HStore(327): Store=1df223c3fe5e10053cc5cd1832f3c22e/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-26T21:52:07,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39637 {}] regionserver.HRegion(1722): Closing 1df223c3fe5e10053cc5cd1832f3c22e, disabling compactions & flushes 2024-11-26T21:52:07,738 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39637 {}] regionserver.HRegion(1755): Closing region testBlockEvictionOnRegionMove1732657925876,,1732657925900.1df223c3fe5e10053cc5cd1832f3c22e. 2024-11-26T21:52:07,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39637 {}] regionserver.HRegion(1776): Time limited wait for close lock on testBlockEvictionOnRegionMove1732657925876,,1732657925900.1df223c3fe5e10053cc5cd1832f3c22e. 2024-11-26T21:52:07,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39637 {}] regionserver.HRegion(1843): Acquired close lock on testBlockEvictionOnRegionMove1732657925876,,1732657925900.1df223c3fe5e10053cc5cd1832f3c22e. after waiting 0 ms 2024-11-26T21:52:07,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39637 {}] regionserver.HRegion(1853): Updates disabled for region testBlockEvictionOnRegionMove1732657925876,,1732657925900.1df223c3fe5e10053cc5cd1832f3c22e. 2024-11-26T21:52:07,739 DEBUG [StoreFileCloser-1df223c3fe5e10053cc5cd1832f3c22e-family-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7430f913-2694-54cf-261c-654b69c5fe06/de866367-466a-3788-a91b-60c9e5aa6ab2/data/default/testBlockEvictionOnRegionMove1732657925876/1df223c3fe5e10053cc5cd1832f3c22e/family/4d545b55745947f0a66d417912877126 2024-11-26T21:52:07,739 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39637 {}] regionserver.HRegion(1973): Closed testBlockEvictionOnRegionMove1732657925876,,1732657925900.1df223c3fe5e10053cc5cd1832f3c22e. 2024-11-26T21:52:07,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39637 {}] regionserver.HRegion(1676): Region close journal for 1df223c3fe5e10053cc5cd1832f3c22e: Waiting for close lock at 1732657927738Disabling compacts and flushes for region at 1732657927738Disabling writes for close at 1732657927738Writing region close event to WAL at 1732657927739 (+1 ms)Closed at 1732657927739 2024-11-26T21:52:07,741 WARN [hfile-prefetch-1732657926695 {}] hfile.HFilePreadReader$1(158): Prefetch path=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7430f913-2694-54cf-261c-654b69c5fe06/de866367-466a-3788-a91b-60c9e5aa6ab2/data/default/testBlockEvictionOnRegionMove1732657925876/1df223c3fe5e10053cc5cd1832f3c22e/family/4d545b55745947f0a66d417912877126, offset=158, end=158 java.lang.RuntimeException: java.lang.InterruptedException: sleep interrupted at org.apache.hadoop.hbase.io.hfile.bucket.BucketCache.notifyFileCachingCompleted(BucketCache.java:2337) ~[classes/:?] at org.apache.hadoop.hbase.io.hfile.CombinedBlockCache.notifyFileCachingCompleted(CombinedBlockCache.java:469) ~[classes/:?] at org.apache.hadoop.hbase.io.hfile.HFilePreadReader$1.run(HFilePreadReader.java:148) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.lambda$tracedRunnable$2(TraceUtil.java:155) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:539) ~[?:?] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:304) ~[?:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCache.notifyFileCachingCompleted(BucketCache.java:2320) ~[classes/:?] ... 9 more 2024-11-26T21:52:07,742 DEBUG [hfile-prefetch-1732657926695 {}] hfile.PrefetchExecutor(121): Prefetch completed for 4d545b55745947f0a66d417912877126 2024-11-26T21:52:07,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36093 {}] procedure.ProcedureSyncWait(219): waitFor pid=7 2024-11-26T21:52:07,893 INFO [RS_CLOSE_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] handler.UnassignRegionHandler(122): Close 1df223c3fe5e10053cc5cd1832f3c22e 2024-11-26T21:52:07,893 DEBUG [RS_CLOSE_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: true 2024-11-26T21:52:07,894 DEBUG [RS_CLOSE_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1722): Closing 1df223c3fe5e10053cc5cd1832f3c22e, disabling compactions & flushes 2024-11-26T21:52:07,894 INFO [RS_CLOSE_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1755): Closing region testBlockEvictionOnRegionMove1732657925876,,1732657925900.1df223c3fe5e10053cc5cd1832f3c22e. 2024-11-26T21:52:07,894 DEBUG [RS_CLOSE_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1776): Time limited wait for close lock on testBlockEvictionOnRegionMove1732657925876,,1732657925900.1df223c3fe5e10053cc5cd1832f3c22e. 2024-11-26T21:52:07,894 DEBUG [RS_CLOSE_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1843): Acquired close lock on testBlockEvictionOnRegionMove1732657925876,,1732657925900.1df223c3fe5e10053cc5cd1832f3c22e. after waiting 0 ms 2024-11-26T21:52:07,894 DEBUG [RS_CLOSE_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1853): Updates disabled for region testBlockEvictionOnRegionMove1732657925876,,1732657925900.1df223c3fe5e10053cc5cd1832f3c22e. 2024-11-26T21:52:07,896 DEBUG [StoreFileCloser-1df223c3fe5e10053cc5cd1832f3c22e-family-1 {}] bucket.BucketCache(1863): found 1 blocks for file 4d545b55745947f0a66d417912877126, starting offset: 0, end offset: 9223372036854775807 2024-11-26T21:52:07,897 DEBUG [StoreFileCloser-1df223c3fe5e10053cc5cd1832f3c22e-family-1 {}] bucket.BucketCache(856): removed key 4d545b55745947f0a66d417912877126_0 from back map with offset lock 2094080 in the evict process 2024-11-26T21:52:07,897 DEBUG [bucket-cache-persister {}] bucket.BucketCachePersister(47): Cache is inconsistent, persisting to disk 2024-11-26T21:52:07,897 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1487): Thread bucket-cache-persister started persisting bucket cache to file 2024-11-26T21:52:07,898 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1494): Persist in new chunked persistence format. 2024-11-26T21:52:07,898 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1673): persistToFile: before persisting backing map size: 0, fullycachedFiles size: 0, chunkSize: 10000 2024-11-26T21:52:07,902 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1680): persistToFile: after persisting backing map size: 0, fullycachedFiles size: 0 2024-11-26T21:52:07,902 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1498): PersistToFile: after persisting backing map size: 0, fullycachedFiles size: 0, file name: bucket.persistence1732657927897 2024-11-26T21:52:07,902 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1509): Thread bucket-cache-persister finished persisting bucket cache to file, renaming 2024-11-26T21:52:07,903 DEBUG [RS_CLOSE_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7430f913-2694-54cf-261c-654b69c5fe06/de866367-466a-3788-a91b-60c9e5aa6ab2/data/default/testBlockEvictionOnRegionMove1732657925876/1df223c3fe5e10053cc5cd1832f3c22e/recovered.edits/9.seqid, newMaxSeqId=9, maxSeqId=1 2024-11-26T21:52:07,903 INFO [RS_CLOSE_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1973): Closed testBlockEvictionOnRegionMove1732657925876,,1732657925900.1df223c3fe5e10053cc5cd1832f3c22e. 2024-11-26T21:52:07,903 DEBUG [RS_CLOSE_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1676): Region close journal for 1df223c3fe5e10053cc5cd1832f3c22e: Waiting for close lock at 1732657927894Running coprocessor pre-close hooks at 1732657927894Disabling compacts and flushes for region at 1732657927894Disabling writes for close at 1732657927894Writing region close event to WAL at 1732657927897 (+3 ms)Running coprocessor post-close hooks at 1732657927903 (+6 ms)Closed at 1732657927903 2024-11-26T21:52:07,904 INFO [RS_CLOSE_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegionServer(3302): Adding 1df223c3fe5e10053cc5cd1832f3c22e move to 828b3f090c91,39637,1732657923848 record at close sequenceid=6 2024-11-26T21:52:07,907 INFO [RS_CLOSE_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] handler.UnassignRegionHandler(157): Closed 1df223c3fe5e10053cc5cd1832f3c22e 2024-11-26T21:52:07,907 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=1df223c3fe5e10053cc5cd1832f3c22e, regionState=CLOSED 2024-11-26T21:52:07,911 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-4-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=8, ppid=7, state=RUNNABLE, hasLock=false; CloseRegionProcedure 1df223c3fe5e10053cc5cd1832f3c22e, server=828b3f090c91,35731,1732657923709 because future has completed 2024-11-26T21:52:07,914 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=8, resume processing ppid=7 2024-11-26T21:52:07,914 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=8, ppid=7, state=SUCCESS, hasLock=false; CloseRegionProcedure 1df223c3fe5e10053cc5cd1832f3c22e, server=828b3f090c91,35731,1732657923709 in 176 msec 2024-11-26T21:52:07,915 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1732657925876, region=1df223c3fe5e10053cc5cd1832f3c22e, REOPEN/MOVE; state=CLOSED, location=828b3f090c91,39637,1732657923848; forceNewPlan=false, retain=false 2024-11-26T21:52:08,066 INFO [828b3f090c91:36093 {}] balancer.BaseLoadBalancer(388): Reassigned 1 regions. 1 retained the pre-restart assignment. 2024-11-26T21:52:08,067 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=1df223c3fe5e10053cc5cd1832f3c22e, regionState=OPENING, regionLocation=828b3f090c91,39637,1732657923848 2024-11-26T21:52:08,073 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-4-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1732657925876, region=1df223c3fe5e10053cc5cd1832f3c22e, REOPEN/MOVE because future has completed 2024-11-26T21:52:08,074 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=9, ppid=7, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1df223c3fe5e10053cc5cd1832f3c22e, server=828b3f090c91,39637,1732657923848}] 2024-11-26T21:52:08,233 INFO [RS_OPEN_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] handler.AssignRegionHandler(132): Open testBlockEvictionOnRegionMove1732657925876,,1732657925900.1df223c3fe5e10053cc5cd1832f3c22e. 2024-11-26T21:52:08,233 DEBUG [RS_OPEN_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(7752): Opening region: {ENCODED => 1df223c3fe5e10053cc5cd1832f3c22e, NAME => 'testBlockEvictionOnRegionMove1732657925876,,1732657925900.1df223c3fe5e10053cc5cd1832f3c22e.', STARTKEY => '', ENDKEY => ''} 2024-11-26T21:52:08,235 DEBUG [RS_OPEN_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testBlockEvictionOnRegionMove1732657925876 1df223c3fe5e10053cc5cd1832f3c22e 2024-11-26T21:52:08,235 DEBUG [RS_OPEN_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(898): Instantiated testBlockEvictionOnRegionMove1732657925876,,1732657925900.1df223c3fe5e10053cc5cd1832f3c22e.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-26T21:52:08,235 DEBUG [RS_OPEN_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(7794): checking encryption for 1df223c3fe5e10053cc5cd1832f3c22e 2024-11-26T21:52:08,235 DEBUG [RS_OPEN_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(7797): checking classloading for 1df223c3fe5e10053cc5cd1832f3c22e 2024-11-26T21:52:08,238 INFO [StoreOpener-1df223c3fe5e10053cc5cd1832f3c22e-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family family of region 1df223c3fe5e10053cc5cd1832f3c22e 2024-11-26T21:52:08,240 INFO [StoreOpener-1df223c3fe5e10053cc5cd1832f3c22e-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1df223c3fe5e10053cc5cd1832f3c22e columnFamilyName family 2024-11-26T21:52:08,240 DEBUG [StoreOpener-1df223c3fe5e10053cc5cd1832f3c22e-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T21:52:08,242 DEBUG [StoreFileOpener-1df223c3fe5e10053cc5cd1832f3c22e-family-1 {}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7430f913-2694-54cf-261c-654b69c5fe06/de866367-466a-3788-a91b-60c9e5aa6ab2/data/default/testBlockEvictionOnRegionMove1732657925876/1df223c3fe5e10053cc5cd1832f3c22e/family/4d545b55745947f0a66d417912877126, delay=929 ms 2024-11-26T21:52:08,244 DEBUG [StoreOpener-1df223c3fe5e10053cc5cd1832f3c22e-1 {}] regionserver.StoreEngine(278): loaded file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7430f913-2694-54cf-261c-654b69c5fe06/de866367-466a-3788-a91b-60c9e5aa6ab2/data/default/testBlockEvictionOnRegionMove1732657925876/1df223c3fe5e10053cc5cd1832f3c22e/family/4d545b55745947f0a66d417912877126 2024-11-26T21:52:08,244 INFO [StoreOpener-1df223c3fe5e10053cc5cd1832f3c22e-1 {}] regionserver.HStore(327): Store=1df223c3fe5e10053cc5cd1832f3c22e/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-26T21:52:08,245 DEBUG [RS_OPEN_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1038): replaying wal for 1df223c3fe5e10053cc5cd1832f3c22e 2024-11-26T21:52:08,246 DEBUG [RS_OPEN_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7430f913-2694-54cf-261c-654b69c5fe06/de866367-466a-3788-a91b-60c9e5aa6ab2/data/default/testBlockEvictionOnRegionMove1732657925876/1df223c3fe5e10053cc5cd1832f3c22e 2024-11-26T21:52:08,246 DEBUG [RS_OPEN_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7430f913-2694-54cf-261c-654b69c5fe06/de866367-466a-3788-a91b-60c9e5aa6ab2/data/default/testBlockEvictionOnRegionMove1732657925876/1df223c3fe5e10053cc5cd1832f3c22e 2024-11-26T21:52:08,246 DEBUG [RS_OPEN_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1048): stopping wal replay for 1df223c3fe5e10053cc5cd1832f3c22e 2024-11-26T21:52:08,246 DEBUG [RS_OPEN_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1060): Cleaning up temporary data for 1df223c3fe5e10053cc5cd1832f3c22e 2024-11-26T21:52:08,250 DEBUG [RS_OPEN_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1093): writing seq id for 1df223c3fe5e10053cc5cd1832f3c22e 2024-11-26T21:52:08,251 INFO [RS_OPEN_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1114): Opened 1df223c3fe5e10053cc5cd1832f3c22e; next sequenceid=10; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=68689609, jitterRate=0.023554936051368713}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-26T21:52:08,251 DEBUG [RS_OPEN_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1df223c3fe5e10053cc5cd1832f3c22e 2024-11-26T21:52:08,252 DEBUG [RS_OPEN_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1006): Region open journal for 1df223c3fe5e10053cc5cd1832f3c22e: Running coprocessor pre-open hook at 1732657928236Writing region info on filesystem at 1732657928236Initializing all the Stores at 1732657928237 (+1 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732657928237Cleaning up temporary data from old regions at 1732657928246 (+9 ms)Running coprocessor post-open hooks at 1732657928251 (+5 ms)Region opened successfully at 1732657928252 (+1 ms) 2024-11-26T21:52:08,253 INFO [RS_OPEN_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegionServer(2236): Post open deploy tasks for testBlockEvictionOnRegionMove1732657925876,,1732657925900.1df223c3fe5e10053cc5cd1832f3c22e., pid=9, masterSystemTime=1732657928227 2024-11-26T21:52:08,256 DEBUG [RS_OPEN_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegionServer(2266): Finished post open deploy task for testBlockEvictionOnRegionMove1732657925876,,1732657925900.1df223c3fe5e10053cc5cd1832f3c22e. 2024-11-26T21:52:08,256 INFO [RS_OPEN_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] handler.AssignRegionHandler(153): Opened testBlockEvictionOnRegionMove1732657925876,,1732657925900.1df223c3fe5e10053cc5cd1832f3c22e. 2024-11-26T21:52:08,256 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=1df223c3fe5e10053cc5cd1832f3c22e, regionState=OPEN, openSeqNum=10, regionLocation=828b3f090c91,39637,1732657923848 2024-11-26T21:52:08,259 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-4-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=9, ppid=7, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1df223c3fe5e10053cc5cd1832f3c22e, server=828b3f090c91,39637,1732657923848 because future has completed 2024-11-26T21:52:08,262 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=9, resume processing ppid=7 2024-11-26T21:52:08,262 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=9, ppid=7, state=SUCCESS, hasLock=false; OpenRegionProcedure 1df223c3fe5e10053cc5cd1832f3c22e, server=828b3f090c91,39637,1732657923848 in 186 msec 2024-11-26T21:52:08,264 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=7, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1732657925876, region=1df223c3fe5e10053cc5cd1832f3c22e, REOPEN/MOVE in 534 msec 2024-11-26T21:52:08,283 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster 2024-11-26T21:52:08,283 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-26T21:52:08,283 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79) at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611) at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.io.hfile.TestBlockEvictionOnRegionMovement.tearDown(TestBlockEvictionOnRegionMovement.java:174) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-26T21:52:08,287 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T21:52:08,287 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T21:52:08,287 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-26T21:52:08,288 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2024-11-26T21:52:08,288 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=1596482003, stopped=false 2024-11-26T21:52:08,288 INFO [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=828b3f090c91,36093,1732657923024 2024-11-26T21:52:08,290 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36093-0x1007e051f2e0000, quorum=127.0.0.1:64198, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-26T21:52:08,290 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39637-0x1007e051f2e0002, quorum=127.0.0.1:64198, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-26T21:52:08,290 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35731-0x1007e051f2e0001, quorum=127.0.0.1:64198, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-26T21:52:08,290 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35731-0x1007e051f2e0001, quorum=127.0.0.1:64198, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-26T21:52:08,290 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39637-0x1007e051f2e0002, quorum=127.0.0.1:64198, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-26T21:52:08,290 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36093-0x1007e051f2e0000, quorum=127.0.0.1:64198, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-26T21:52:08,290 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-26T21:52:08,291 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-26T21:52:08,291 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:35731-0x1007e051f2e0001, quorum=127.0.0.1:64198, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-26T21:52:08,291 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277) at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265) at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416) at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.io.hfile.TestBlockEvictionOnRegionMovement.tearDown(TestBlockEvictionOnRegionMovement.java:174) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-26T21:52:08,291 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T21:52:08,292 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:39637-0x1007e051f2e0002, quorum=127.0.0.1:64198, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-26T21:52:08,292 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '828b3f090c91,35731,1732657923709' ***** 2024-11-26T21:52:08,292 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-11-26T21:52:08,292 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:36093-0x1007e051f2e0000, quorum=127.0.0.1:64198, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-26T21:52:08,292 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '828b3f090c91,39637,1732657923848' ***** 2024-11-26T21:52:08,292 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-11-26T21:52:08,292 INFO [RS:0;828b3f090c91:35731 {}] bucket.BucketCache(1738): Disabling cache 2024-11-26T21:52:08,292 INFO [RS:1;828b3f090c91:39637 {}] bucket.BucketCache(1738): Disabling cache 2024-11-26T21:52:08,293 INFO [RS:1;828b3f090c91:39637 {}] bucket.BucketCache(1762): Shutdown bucket cache: IO persistent=true; path to write=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7430f913-2694-54cf-261c-654b69c5fe06/bucket.persistence 2024-11-26T21:52:08,293 INFO [Time-limited test-BucketCacheWriter-1 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-1 exiting, cacheEnabled=false 2024-11-26T21:52:08,293 INFO [Time-limited test-BucketCacheWriter-2 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-2 exiting, cacheEnabled=false 2024-11-26T21:52:08,293 INFO [Time-limited test-BucketCacheWriter-0 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-0 exiting, cacheEnabled=false 2024-11-26T21:52:08,293 INFO [RS:0;828b3f090c91:35731 {}] bucket.BucketCache(1762): Shutdown bucket cache: IO persistent=true; path to write=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7430f913-2694-54cf-261c-654b69c5fe06/bucket.persistence 2024-11-26T21:52:08,293 INFO [Time-limited test-BucketCacheWriter-0 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-0 exiting, cacheEnabled=false 2024-11-26T21:52:08,293 INFO [Time-limited test-BucketCacheWriter-2 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-2 exiting, cacheEnabled=false 2024-11-26T21:52:08,293 INFO [Time-limited test-BucketCacheWriter-1 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-1 exiting, cacheEnabled=false 2024-11-26T21:52:08,293 INFO [RS:1;828b3f090c91:39637 {}] bucket.BucketCache(1768): Shutting down cache persister thread. 2024-11-26T21:52:08,293 INFO [RS:0;828b3f090c91:35731 {}] bucket.BucketCache(1768): Shutting down cache persister thread. 2024-11-26T21:52:08,293 WARN [bucket-cache-persister {}] bucket.BucketCachePersister(64): Interrupting BucketCachePersister thread. java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCachePersister.run(BucketCachePersister.java:45) ~[classes/:?] 2024-11-26T21:52:08,293 WARN [bucket-cache-persister {}] bucket.BucketCachePersister(64): Interrupting BucketCachePersister thread. java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCachePersister.run(BucketCachePersister.java:45) ~[classes/:?] 2024-11-26T21:52:08,304 DEBUG [RS:1;828b3f090c91:39637 {}] bucket.BucketCache(1487): Thread RS:1;828b3f090c91:39637 started persisting bucket cache to file 2024-11-26T21:52:08,304 DEBUG [RS:0;828b3f090c91:35731 {}] bucket.BucketCache(1487): Thread RS:0;828b3f090c91:35731 started persisting bucket cache to file 2024-11-26T21:52:08,304 DEBUG [RS:0;828b3f090c91:35731 {}] bucket.BucketCache(1494): Persist in new chunked persistence format. 2024-11-26T21:52:08,304 DEBUG [RS:1;828b3f090c91:39637 {}] bucket.BucketCache(1494): Persist in new chunked persistence format. 2024-11-26T21:52:08,304 DEBUG [RS:0;828b3f090c91:35731 {}] bucket.BucketCache(1673): persistToFile: before persisting backing map size: 0, fullycachedFiles size: 0, chunkSize: 10000 2024-11-26T21:52:08,304 DEBUG [RS:1;828b3f090c91:39637 {}] bucket.BucketCache(1673): persistToFile: before persisting backing map size: 0, fullycachedFiles size: 0, chunkSize: 10000 2024-11-26T21:52:08,306 DEBUG [RS:0;828b3f090c91:35731 {}] bucket.BucketCache(1680): persistToFile: after persisting backing map size: 0, fullycachedFiles size: 0 2024-11-26T21:52:08,306 DEBUG [RS:0;828b3f090c91:35731 {}] bucket.BucketCache(1498): PersistToFile: after persisting backing map size: 0, fullycachedFiles size: 0, file name: bucket.persistence1732657928304 2024-11-26T21:52:08,307 DEBUG [RS:1;828b3f090c91:39637 {}] bucket.BucketCache(1680): persistToFile: after persisting backing map size: 0, fullycachedFiles size: 0 2024-11-26T21:52:08,307 DEBUG [RS:0;828b3f090c91:35731 {}] bucket.BucketCache(1509): Thread RS:0;828b3f090c91:35731 finished persisting bucket cache to file, renaming 2024-11-26T21:52:08,307 DEBUG [RS:1;828b3f090c91:39637 {}] bucket.BucketCache(1498): PersistToFile: after persisting backing map size: 0, fullycachedFiles size: 0, file name: bucket.persistence1732657928304 2024-11-26T21:52:08,307 DEBUG [RS:1;828b3f090c91:39637 {}] bucket.BucketCache(1509): Thread RS:1;828b3f090c91:39637 finished persisting bucket cache to file, renaming 2024-11-26T21:52:08,307 WARN [RS:1;828b3f090c91:39637 {}] bucket.BucketCache(1512): Failed to commit cache persistent file. We might lose cached blocks if RS crashes/restarts before we successfully checkpoint again. 2024-11-26T21:52:08,307 INFO [RS:1;828b3f090c91:39637 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-26T21:52:08,307 INFO [RS:0;828b3f090c91:35731 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-26T21:52:08,307 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-26T21:52:08,307 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-26T21:52:08,307 INFO [RS:1;828b3f090c91:39637 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-26T21:52:08,307 INFO [RS:0;828b3f090c91:35731 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-26T21:52:08,307 INFO [RS:1;828b3f090c91:39637 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-26T21:52:08,307 INFO [RS:0;828b3f090c91:35731 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-26T21:52:08,307 INFO [RS:0;828b3f090c91:35731 {}] regionserver.HRegionServer(959): stopping server 828b3f090c91,35731,1732657923709 2024-11-26T21:52:08,307 INFO [RS:1;828b3f090c91:39637 {}] regionserver.HRegionServer(3091): Received CLOSE for 1df223c3fe5e10053cc5cd1832f3c22e 2024-11-26T21:52:08,307 INFO [RS:0;828b3f090c91:35731 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-26T21:52:08,307 INFO [RS:0;828b3f090c91:35731 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;828b3f090c91:35731. 2024-11-26T21:52:08,308 DEBUG [RS:0;828b3f090c91:35731 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-26T21:52:08,308 DEBUG [RS:0;828b3f090c91:35731 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T21:52:08,308 INFO [RS:1;828b3f090c91:39637 {}] regionserver.HRegionServer(959): stopping server 828b3f090c91,39637,1732657923848 2024-11-26T21:52:08,308 INFO [RS:0;828b3f090c91:35731 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-26T21:52:08,308 INFO [RS:1;828b3f090c91:39637 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-26T21:52:08,308 INFO [RS:0;828b3f090c91:35731 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-26T21:52:08,308 INFO [RS:0;828b3f090c91:35731 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-26T21:52:08,308 INFO [RS:1;828b3f090c91:39637 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:1;828b3f090c91:39637. 2024-11-26T21:52:08,308 INFO [RS:0;828b3f090c91:35731 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-11-26T21:52:08,308 DEBUG [RS:1;828b3f090c91:39637 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-26T21:52:08,308 DEBUG [RS:1;828b3f090c91:39637 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T21:52:08,308 INFO [RS:1;828b3f090c91:39637 {}] regionserver.HRegionServer(1321): Waiting on 1 regions to close 2024-11-26T21:52:08,308 INFO [RS:0;828b3f090c91:35731 {}] regionserver.HRegionServer(1321): Waiting on 1 regions to close 2024-11-26T21:52:08,308 INFO [RS_CLOSE_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1662): Closing region testBlockEvictionOnRegionMove1732657925876,,1732657925900.1df223c3fe5e10053cc5cd1832f3c22e. during a graceful stop, and cache persistence is on, so setting evict on close to false. 2024-11-26T21:52:08,308 DEBUG [RS:0;828b3f090c91:35731 {}] regionserver.HRegionServer(1325): Online Regions={1588230740=hbase:meta,,1.1588230740} 2024-11-26T21:52:08,308 DEBUG [RS:1;828b3f090c91:39637 {}] regionserver.HRegionServer(1325): Online Regions={1df223c3fe5e10053cc5cd1832f3c22e=testBlockEvictionOnRegionMove1732657925876,,1732657925900.1df223c3fe5e10053cc5cd1832f3c22e.} 2024-11-26T21:52:08,309 INFO [RS_CLOSE_META-regionserver/828b3f090c91:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1662): Closing region hbase:meta,,1.1588230740 during a graceful stop, and cache persistence is on, so setting evict on close to false. 2024-11-26T21:52:08,309 DEBUG [RS:0;828b3f090c91:35731 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-26T21:52:08,309 DEBUG [RS_CLOSE_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 1df223c3fe5e10053cc5cd1832f3c22e, disabling compactions & flushes 2024-11-26T21:52:08,309 DEBUG [RS:1;828b3f090c91:39637 {}] regionserver.HRegionServer(1351): Waiting on 1df223c3fe5e10053cc5cd1832f3c22e 2024-11-26T21:52:08,309 INFO [RS_CLOSE_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testBlockEvictionOnRegionMove1732657925876,,1732657925900.1df223c3fe5e10053cc5cd1832f3c22e. 2024-11-26T21:52:08,309 DEBUG [RS_CLOSE_META-regionserver/828b3f090c91:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-26T21:52:08,309 DEBUG [RS_CLOSE_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testBlockEvictionOnRegionMove1732657925876,,1732657925900.1df223c3fe5e10053cc5cd1832f3c22e. 2024-11-26T21:52:08,309 DEBUG [RS_CLOSE_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testBlockEvictionOnRegionMove1732657925876,,1732657925900.1df223c3fe5e10053cc5cd1832f3c22e. after waiting 0 ms 2024-11-26T21:52:08,309 INFO [RS_CLOSE_META-regionserver/828b3f090c91:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-26T21:52:08,309 DEBUG [RS_CLOSE_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testBlockEvictionOnRegionMove1732657925876,,1732657925900.1df223c3fe5e10053cc5cd1832f3c22e. 2024-11-26T21:52:08,309 DEBUG [RS_CLOSE_META-regionserver/828b3f090c91:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-26T21:52:08,309 DEBUG [RS_CLOSE_META-regionserver/828b3f090c91:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-26T21:52:08,309 DEBUG [RS_CLOSE_META-regionserver/828b3f090c91:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-26T21:52:08,310 DEBUG [StoreFileCloser-1df223c3fe5e10053cc5cd1832f3c22e-family-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7430f913-2694-54cf-261c-654b69c5fe06/de866367-466a-3788-a91b-60c9e5aa6ab2/data/default/testBlockEvictionOnRegionMove1732657925876/1df223c3fe5e10053cc5cd1832f3c22e/family/4d545b55745947f0a66d417912877126 2024-11-26T21:52:08,310 INFO [RS_CLOSE_META-regionserver/828b3f090c91:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=3.70 KB heapSize=6.72 KB 2024-11-26T21:52:08,314 DEBUG [RS_CLOSE_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7430f913-2694-54cf-261c-654b69c5fe06/de866367-466a-3788-a91b-60c9e5aa6ab2/data/default/testBlockEvictionOnRegionMove1732657925876/1df223c3fe5e10053cc5cd1832f3c22e/recovered.edits/12.seqid, newMaxSeqId=12, maxSeqId=9 2024-11-26T21:52:08,315 INFO [RS_CLOSE_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testBlockEvictionOnRegionMove1732657925876,,1732657925900.1df223c3fe5e10053cc5cd1832f3c22e. 2024-11-26T21:52:08,315 DEBUG [RS_CLOSE_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 1df223c3fe5e10053cc5cd1832f3c22e: Waiting for close lock at 1732657928308Running coprocessor pre-close hooks at 1732657928309 (+1 ms)Disabling compacts and flushes for region at 1732657928309Disabling writes for close at 1732657928309Writing region close event to WAL at 1732657928310 (+1 ms)Running coprocessor post-close hooks at 1732657928315 (+5 ms)Closed at 1732657928315 2024-11-26T21:52:08,315 DEBUG [RS_CLOSE_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testBlockEvictionOnRegionMove1732657925876,,1732657925900.1df223c3fe5e10053cc5cd1832f3c22e. 2024-11-26T21:52:08,320 DEBUG [RS_CLOSE_META-regionserver/828b3f090c91:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 17e57b3d90d54435bf5d205f5c389aa5 2024-11-26T21:52:08,337 INFO [regionserver/828b3f090c91:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-26T21:52:08,337 DEBUG [RS_CLOSE_META-regionserver/828b3f090c91:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7430f913-2694-54cf-261c-654b69c5fe06/de866367-466a-3788-a91b-60c9e5aa6ab2/data/hbase/meta/1588230740/.tmp/info/17e57b3d90d54435bf5d205f5c389aa5 is 205, key is testBlockEvictionOnRegionMove1732657925876,,1732657925900.1df223c3fe5e10053cc5cd1832f3c22e./info:regioninfo/1732657928256/Put/seqid=0 2024-11-26T21:52:08,338 INFO [RS_CLOSE_META-regionserver/828b3f090c91:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=3.48 KB at sequenceid=15 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7430f913-2694-54cf-261c-654b69c5fe06/de866367-466a-3788-a91b-60c9e5aa6ab2/data/hbase/meta/1588230740/.tmp/info/17e57b3d90d54435bf5d205f5c389aa5 2024-11-26T21:52:08,340 INFO [regionserver/828b3f090c91:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-26T21:52:08,348 DEBUG [RS_CLOSE_META-regionserver/828b3f090c91:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFile$WriterFactory(308): Unable to set drop behind on c7275f3d4cbc4e97bda327826ecdbee5 2024-11-26T21:52:08,361 DEBUG [RS_CLOSE_META-regionserver/828b3f090c91:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7430f913-2694-54cf-261c-654b69c5fe06/de866367-466a-3788-a91b-60c9e5aa6ab2/data/hbase/meta/1588230740/.tmp/ns/c7275f3d4cbc4e97bda327826ecdbee5 is 43, key is default/ns:d/1732657925625/Put/seqid=0 2024-11-26T21:52:08,362 INFO [RS_CLOSE_META-regionserver/828b3f090c91:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=74 B at sequenceid=15 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7430f913-2694-54cf-261c-654b69c5fe06/de866367-466a-3788-a91b-60c9e5aa6ab2/data/hbase/meta/1588230740/.tmp/ns/c7275f3d4cbc4e97bda327826ecdbee5 2024-11-26T21:52:08,369 DEBUG [RS_CLOSE_META-regionserver/828b3f090c91:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 83167be8a1fd4fa38996df02eb281881 2024-11-26T21:52:08,382 DEBUG [RS_CLOSE_META-regionserver/828b3f090c91:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7430f913-2694-54cf-261c-654b69c5fe06/de866367-466a-3788-a91b-60c9e5aa6ab2/data/hbase/meta/1588230740/.tmp/table/83167be8a1fd4fa38996df02eb281881 is 78, key is testBlockEvictionOnRegionMove1732657925876/table:state/1732657926317/Put/seqid=0 2024-11-26T21:52:08,382 INFO [RS_CLOSE_META-regionserver/828b3f090c91:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=148 B at sequenceid=15 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7430f913-2694-54cf-261c-654b69c5fe06/de866367-466a-3788-a91b-60c9e5aa6ab2/data/hbase/meta/1588230740/.tmp/table/83167be8a1fd4fa38996df02eb281881 2024-11-26T21:52:08,385 DEBUG [RS_CLOSE_META-regionserver/828b3f090c91:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7430f913-2694-54cf-261c-654b69c5fe06/de866367-466a-3788-a91b-60c9e5aa6ab2/data/hbase/meta/1588230740/.tmp/info/17e57b3d90d54435bf5d205f5c389aa5 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7430f913-2694-54cf-261c-654b69c5fe06/de866367-466a-3788-a91b-60c9e5aa6ab2/data/hbase/meta/1588230740/info/17e57b3d90d54435bf5d205f5c389aa5 2024-11-26T21:52:08,386 DEBUG [RS_CLOSE_META-regionserver/828b3f090c91:0-0 {event_type=M_RS_CLOSE_META}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7430f913-2694-54cf-261c-654b69c5fe06/de866367-466a-3788-a91b-60c9e5aa6ab2/data/hbase/meta/1588230740/info/17e57b3d90d54435bf5d205f5c389aa5, delay=906 ms 2024-11-26T21:52:08,387 INFO [RS_CLOSE_META-regionserver/828b3f090c91:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7430f913-2694-54cf-261c-654b69c5fe06/de866367-466a-3788-a91b-60c9e5aa6ab2/data/hbase/meta/1588230740/info/17e57b3d90d54435bf5d205f5c389aa5, entries=15, sequenceid=15, filesize=7.6 K 2024-11-26T21:52:08,388 DEBUG [RS_CLOSE_META-regionserver/828b3f090c91:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7430f913-2694-54cf-261c-654b69c5fe06/de866367-466a-3788-a91b-60c9e5aa6ab2/data/hbase/meta/1588230740/.tmp/ns/c7275f3d4cbc4e97bda327826ecdbee5 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7430f913-2694-54cf-261c-654b69c5fe06/de866367-466a-3788-a91b-60c9e5aa6ab2/data/hbase/meta/1588230740/ns/c7275f3d4cbc4e97bda327826ecdbee5 2024-11-26T21:52:08,389 DEBUG [RS_CLOSE_META-regionserver/828b3f090c91:0-0 {event_type=M_RS_CLOSE_META}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7430f913-2694-54cf-261c-654b69c5fe06/de866367-466a-3788-a91b-60c9e5aa6ab2/data/hbase/meta/1588230740/ns/c7275f3d4cbc4e97bda327826ecdbee5, delay=973 ms 2024-11-26T21:52:08,390 INFO [RS_CLOSE_META-regionserver/828b3f090c91:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7430f913-2694-54cf-261c-654b69c5fe06/de866367-466a-3788-a91b-60c9e5aa6ab2/data/hbase/meta/1588230740/ns/c7275f3d4cbc4e97bda327826ecdbee5, entries=2, sequenceid=15, filesize=5.0 K 2024-11-26T21:52:08,391 DEBUG [RS_CLOSE_META-regionserver/828b3f090c91:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7430f913-2694-54cf-261c-654b69c5fe06/de866367-466a-3788-a91b-60c9e5aa6ab2/data/hbase/meta/1588230740/.tmp/table/83167be8a1fd4fa38996df02eb281881 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7430f913-2694-54cf-261c-654b69c5fe06/de866367-466a-3788-a91b-60c9e5aa6ab2/data/hbase/meta/1588230740/table/83167be8a1fd4fa38996df02eb281881 2024-11-26T21:52:08,392 DEBUG [RS_CLOSE_META-regionserver/828b3f090c91:0-0 {event_type=M_RS_CLOSE_META}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7430f913-2694-54cf-261c-654b69c5fe06/de866367-466a-3788-a91b-60c9e5aa6ab2/data/hbase/meta/1588230740/table/83167be8a1fd4fa38996df02eb281881, delay=978 ms 2024-11-26T21:52:08,393 INFO [RS_CLOSE_META-regionserver/828b3f090c91:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7430f913-2694-54cf-261c-654b69c5fe06/de866367-466a-3788-a91b-60c9e5aa6ab2/data/hbase/meta/1588230740/table/83167be8a1fd4fa38996df02eb281881, entries=2, sequenceid=15, filesize=5.3 K 2024-11-26T21:52:08,394 INFO [RS_CLOSE_META-regionserver/828b3f090c91:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~3.70 KB/3789, heapSize ~6.42 KB/6576, currentSize=0 B/0 for 1588230740 in 84ms, sequenceid=15, compaction requested=false 2024-11-26T21:52:08,394 DEBUG [RS_CLOSE_META-regionserver/828b3f090c91:0-0 {event_type=M_RS_CLOSE_META}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta' 2024-11-26T21:52:08,395 DEBUG [StoreFileCloser-1588230740-info-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7430f913-2694-54cf-261c-654b69c5fe06/de866367-466a-3788-a91b-60c9e5aa6ab2/data/hbase/meta/1588230740/info/17e57b3d90d54435bf5d205f5c389aa5 2024-11-26T21:52:08,396 DEBUG [StoreFileCloser-1588230740-ns-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7430f913-2694-54cf-261c-654b69c5fe06/de866367-466a-3788-a91b-60c9e5aa6ab2/data/hbase/meta/1588230740/ns/c7275f3d4cbc4e97bda327826ecdbee5 2024-11-26T21:52:08,396 DEBUG [StoreFileCloser-1588230740-table-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7430f913-2694-54cf-261c-654b69c5fe06/de866367-466a-3788-a91b-60c9e5aa6ab2/data/hbase/meta/1588230740/table/83167be8a1fd4fa38996df02eb281881 2024-11-26T21:52:08,401 DEBUG [RS_CLOSE_META-regionserver/828b3f090c91:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7430f913-2694-54cf-261c-654b69c5fe06/de866367-466a-3788-a91b-60c9e5aa6ab2/data/hbase/meta/1588230740/recovered.edits/18.seqid, newMaxSeqId=18, maxSeqId=1 2024-11-26T21:52:08,401 DEBUG [RS_CLOSE_META-regionserver/828b3f090c91:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-26T21:52:08,402 INFO [RS_CLOSE_META-regionserver/828b3f090c91:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-26T21:52:08,402 DEBUG [RS_CLOSE_META-regionserver/828b3f090c91:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1732657928308Running coprocessor pre-close hooks at 1732657928309 (+1 ms)Disabling compacts and flushes for region at 1732657928309Disabling writes for close at 1732657928309Obtaining lock to block concurrent updates at 1732657928310 (+1 ms)Preparing flush snapshotting stores in 1588230740 at 1732657928310Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=3789, getHeapSize=6816, getOffHeapSize=0, getCellsCount=27 at 1732657928310Flushing stores of hbase:meta,,1.1588230740 at 1732657928311 (+1 ms)Flushing 1588230740/info: creating writer at 1732657928311Flushing 1588230740/info: appending metadata at 1732657928336 (+25 ms)Flushing 1588230740/info: closing flushed file at 1732657928336Flushing 1588230740/ns: creating writer at 1732657928341 (+5 ms)Flushing 1588230740/ns: appending metadata at 1732657928360 (+19 ms)Flushing 1588230740/ns: closing flushed file at 1732657928360Flushing 1588230740/table: creating writer at 1732657928364 (+4 ms)Flushing 1588230740/table: appending metadata at 1732657928381 (+17 ms)Flushing 1588230740/table: closing flushed file at 1732657928381Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5d5873d7: reopening flushed file at 1732657928385 (+4 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5f6f0eb1: reopening flushed file at 1732657928388 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@24d87b9a: reopening flushed file at 1732657928390 (+2 ms)Finished flush of dataSize ~3.70 KB/3789, heapSize ~6.42 KB/6576, currentSize=0 B/0 for 1588230740 in 84ms, sequenceid=15, compaction requested=false at 1732657928394 (+4 ms)Writing region close event to WAL at 1732657928397 (+3 ms)Running coprocessor post-close hooks at 1732657928401 (+4 ms)Closed at 1732657928402 (+1 ms) 2024-11-26T21:52:08,402 DEBUG [RS_CLOSE_META-regionserver/828b3f090c91:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-11-26T21:52:08,509 INFO [RS:0;828b3f090c91:35731 {}] regionserver.HRegionServer(976): stopping server 828b3f090c91,35731,1732657923709; all regions closed. 2024-11-26T21:52:08,509 INFO [RS:1;828b3f090c91:39637 {}] regionserver.HRegionServer(976): stopping server 828b3f090c91,39637,1732657923848; all regions closed. 2024-11-26T21:52:08,515 DEBUG [RS:1;828b3f090c91:39637 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7430f913-2694-54cf-261c-654b69c5fe06/de866367-466a-3788-a91b-60c9e5aa6ab2/oldWALs 2024-11-26T21:52:08,515 DEBUG [RS:0;828b3f090c91:35731 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7430f913-2694-54cf-261c-654b69c5fe06/de866367-466a-3788-a91b-60c9e5aa6ab2/oldWALs 2024-11-26T21:52:08,516 INFO [RS:0;828b3f090c91:35731 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 828b3f090c91%2C35731%2C1732657923709.meta:.meta(num 1732657925499) 2024-11-26T21:52:08,516 INFO [RS:1;828b3f090c91:39637 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 828b3f090c91%2C39637%2C1732657923848:(num 1732657925097) 2024-11-26T21:52:08,516 DEBUG [RS:1;828b3f090c91:39637 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T21:52:08,516 INFO [RS:1;828b3f090c91:39637 {}] regionserver.LeaseManager(133): Closed leases 2024-11-26T21:52:08,516 INFO [RS:1;828b3f090c91:39637 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-26T21:52:08,516 INFO [RS:1;828b3f090c91:39637 {}] hbase.ChoreService(370): Chore service for: regionserver/828b3f090c91:0 had [ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS] on shutdown 2024-11-26T21:52:08,517 INFO [RS:1;828b3f090c91:39637 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-26T21:52:08,517 INFO [regionserver/828b3f090c91:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-26T21:52:08,517 INFO [RS:1;828b3f090c91:39637 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-26T21:52:08,517 INFO [RS:1;828b3f090c91:39637 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-26T21:52:08,517 INFO [RS:1;828b3f090c91:39637 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-26T21:52:08,518 INFO [RS:1;828b3f090c91:39637 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:39637 2024-11-26T21:52:08,518 DEBUG [RS:0;828b3f090c91:35731 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7430f913-2694-54cf-261c-654b69c5fe06/de866367-466a-3788-a91b-60c9e5aa6ab2/oldWALs 2024-11-26T21:52:08,518 INFO [RS:0;828b3f090c91:35731 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 828b3f090c91%2C35731%2C1732657923709:(num 1732657925097) 2024-11-26T21:52:08,518 DEBUG [RS:0;828b3f090c91:35731 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T21:52:08,519 INFO [RS:0;828b3f090c91:35731 {}] regionserver.LeaseManager(133): Closed leases 2024-11-26T21:52:08,519 INFO [RS:0;828b3f090c91:35731 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-26T21:52:08,519 INFO [RS:0;828b3f090c91:35731 {}] hbase.ChoreService(370): Chore service for: regionserver/828b3f090c91:0 had [ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-11-26T21:52:08,520 INFO [RS:0;828b3f090c91:35731 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-26T21:52:08,520 INFO [regionserver/828b3f090c91:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-26T21:52:08,520 INFO [RS:0;828b3f090c91:35731 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:35731 2024-11-26T21:52:08,522 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36093-0x1007e051f2e0000, quorum=127.0.0.1:64198, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-26T21:52:08,523 INFO [RS:1;828b3f090c91:39637 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-26T21:52:08,523 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39637-0x1007e051f2e0002, quorum=127.0.0.1:64198, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/828b3f090c91,39637,1732657923848 2024-11-26T21:52:08,523 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35731-0x1007e051f2e0001, quorum=127.0.0.1:64198, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/828b3f090c91,35731,1732657923709 2024-11-26T21:52:08,523 INFO [RS:0;828b3f090c91:35731 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-26T21:52:08,524 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [828b3f090c91,39637,1732657923848] 2024-11-26T21:52:08,526 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/828b3f090c91,39637,1732657923848 already deleted, retry=false 2024-11-26T21:52:08,526 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; 828b3f090c91,39637,1732657923848 expired; onlineServers=1 2024-11-26T21:52:08,526 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [828b3f090c91,35731,1732657923709] 2024-11-26T21:52:08,527 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/828b3f090c91,35731,1732657923709 already deleted, retry=false 2024-11-26T21:52:08,527 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; 828b3f090c91,35731,1732657923709 expired; onlineServers=0 2024-11-26T21:52:08,527 INFO [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master '828b3f090c91,36093,1732657923024' ***** 2024-11-26T21:52:08,527 INFO [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0 2024-11-26T21:52:08,528 INFO [M:0;828b3f090c91:36093 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-26T21:52:08,528 INFO [M:0;828b3f090c91:36093 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-26T21:52:08,528 DEBUG [M:0;828b3f090c91:36093 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2024-11-26T21:52:08,528 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2024-11-26T21:52:08,528 DEBUG [M:0;828b3f090c91:36093 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2024-11-26T21:52:08,528 DEBUG [master/828b3f090c91:0:becomeActiveMaster-HFileCleaner.large.0-1732657924775 {}] cleaner.HFileCleaner(306): Exit Thread[master/828b3f090c91:0:becomeActiveMaster-HFileCleaner.large.0-1732657924775,5,FailOnTimeoutGroup] 2024-11-26T21:52:08,528 DEBUG [master/828b3f090c91:0:becomeActiveMaster-HFileCleaner.small.0-1732657924776 {}] cleaner.HFileCleaner(306): Exit Thread[master/828b3f090c91:0:becomeActiveMaster-HFileCleaner.small.0-1732657924776,5,FailOnTimeoutGroup] 2024-11-26T21:52:08,528 INFO [M:0;828b3f090c91:36093 {}] hbase.ChoreService(370): Chore service for: master/828b3f090c91:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown 2024-11-26T21:52:08,529 INFO [M:0;828b3f090c91:36093 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-26T21:52:08,529 DEBUG [M:0;828b3f090c91:36093 {}] master.HMaster(1795): Stopping service threads 2024-11-26T21:52:08,529 INFO [M:0;828b3f090c91:36093 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2024-11-26T21:52:08,529 INFO [M:0;828b3f090c91:36093 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-26T21:52:08,529 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36093-0x1007e051f2e0000, quorum=127.0.0.1:64198, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-11-26T21:52:08,529 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36093-0x1007e051f2e0000, quorum=127.0.0.1:64198, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-26T21:52:08,530 INFO [M:0;828b3f090c91:36093 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-26T21:52:08,530 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2024-11-26T21:52:08,530 DEBUG [M:0;828b3f090c91:36093 {}] zookeeper.ZKUtil(347): master:36093-0x1007e051f2e0000, quorum=127.0.0.1:64198, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error) 2024-11-26T21:52:08,530 WARN [M:0;828b3f090c91:36093 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null 2024-11-26T21:52:08,531 INFO [M:0;828b3f090c91:36093 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7430f913-2694-54cf-261c-654b69c5fe06/de866367-466a-3788-a91b-60c9e5aa6ab2/.lastflushedseqids 2024-11-26T21:52:08,538 INFO [M:0;828b3f090c91:36093 {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-26T21:52:08,538 INFO [M:0;828b3f090c91:36093 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2024-11-26T21:52:08,538 DEBUG [M:0;828b3f090c91:36093 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-26T21:52:08,538 INFO [M:0;828b3f090c91:36093 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-26T21:52:08,538 DEBUG [M:0;828b3f090c91:36093 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-26T21:52:08,538 DEBUG [M:0;828b3f090c91:36093 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-26T21:52:08,538 DEBUG [M:0;828b3f090c91:36093 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-26T21:52:08,538 INFO [M:0;828b3f090c91:36093 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=34.67 KB heapSize=43.02 KB 2024-11-26T21:52:08,547 DEBUG [M:0;828b3f090c91:36093 {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 0e54e82a8c1d4e039a7c8268af7fde0e 2024-11-26T21:52:08,560 DEBUG [M:0;828b3f090c91:36093 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7430f913-2694-54cf-261c-654b69c5fe06/de866367-466a-3788-a91b-60c9e5aa6ab2/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/0e54e82a8c1d4e039a7c8268af7fde0e is 82, key is hbase:meta,,1/info:regioninfo/1732657925568/Put/seqid=0 2024-11-26T21:52:08,560 INFO [M:0;828b3f090c91:36093 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=504 B at sequenceid=84 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7430f913-2694-54cf-261c-654b69c5fe06/de866367-466a-3788-a91b-60c9e5aa6ab2/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/0e54e82a8c1d4e039a7c8268af7fde0e 2024-11-26T21:52:08,568 DEBUG [M:0;828b3f090c91:36093 {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on cbfb88f3e9974211bc2cd847ce408442 2024-11-26T21:52:08,580 DEBUG [M:0;828b3f090c91:36093 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7430f913-2694-54cf-261c-654b69c5fe06/de866367-466a-3788-a91b-60c9e5aa6ab2/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/cbfb88f3e9974211bc2cd847ce408442 is 778, key is \x00\x00\x00\x00\x00\x00\x00\x04/proc:d/1732657926322/Put/seqid=0 2024-11-26T21:52:08,580 INFO [M:0;828b3f090c91:36093 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=34.05 KB at sequenceid=84 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7430f913-2694-54cf-261c-654b69c5fe06/de866367-466a-3788-a91b-60c9e5aa6ab2/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/cbfb88f3e9974211bc2cd847ce408442 2024-11-26T21:52:08,587 DEBUG [M:0;828b3f090c91:36093 {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 3f5e538592a840a49c30c05c5b039157 2024-11-26T21:52:08,598 DEBUG [M:0;828b3f090c91:36093 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7430f913-2694-54cf-261c-654b69c5fe06/de866367-466a-3788-a91b-60c9e5aa6ab2/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/3f5e538592a840a49c30c05c5b039157 is 69, key is 828b3f090c91,35731,1732657923709/rs:state/1732657924846/Put/seqid=0 2024-11-26T21:52:08,599 INFO [M:0;828b3f090c91:36093 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=130 B at sequenceid=84 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7430f913-2694-54cf-261c-654b69c5fe06/de866367-466a-3788-a91b-60c9e5aa6ab2/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/3f5e538592a840a49c30c05c5b039157 2024-11-26T21:52:08,601 DEBUG [M:0;828b3f090c91:36093 {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7430f913-2694-54cf-261c-654b69c5fe06/de866367-466a-3788-a91b-60c9e5aa6ab2/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/0e54e82a8c1d4e039a7c8268af7fde0e as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7430f913-2694-54cf-261c-654b69c5fe06/de866367-466a-3788-a91b-60c9e5aa6ab2/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/0e54e82a8c1d4e039a7c8268af7fde0e 2024-11-26T21:52:08,602 INFO [M:0;828b3f090c91:36093 {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7430f913-2694-54cf-261c-654b69c5fe06/de866367-466a-3788-a91b-60c9e5aa6ab2/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/0e54e82a8c1d4e039a7c8268af7fde0e, entries=8, sequenceid=84, filesize=5.5 K 2024-11-26T21:52:08,603 DEBUG [M:0;828b3f090c91:36093 {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7430f913-2694-54cf-261c-654b69c5fe06/de866367-466a-3788-a91b-60c9e5aa6ab2/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/cbfb88f3e9974211bc2cd847ce408442 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7430f913-2694-54cf-261c-654b69c5fe06/de866367-466a-3788-a91b-60c9e5aa6ab2/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/cbfb88f3e9974211bc2cd847ce408442 2024-11-26T21:52:08,604 INFO [M:0;828b3f090c91:36093 {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7430f913-2694-54cf-261c-654b69c5fe06/de866367-466a-3788-a91b-60c9e5aa6ab2/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/cbfb88f3e9974211bc2cd847ce408442, entries=9, sequenceid=84, filesize=6.5 K 2024-11-26T21:52:08,604 DEBUG [M:0;828b3f090c91:36093 {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7430f913-2694-54cf-261c-654b69c5fe06/de866367-466a-3788-a91b-60c9e5aa6ab2/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/3f5e538592a840a49c30c05c5b039157 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7430f913-2694-54cf-261c-654b69c5fe06/de866367-466a-3788-a91b-60c9e5aa6ab2/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/3f5e538592a840a49c30c05c5b039157 2024-11-26T21:52:08,606 INFO [M:0;828b3f090c91:36093 {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7430f913-2694-54cf-261c-654b69c5fe06/de866367-466a-3788-a91b-60c9e5aa6ab2/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/3f5e538592a840a49c30c05c5b039157, entries=2, sequenceid=84, filesize=5.1 K 2024-11-26T21:52:08,606 INFO [M:0;828b3f090c91:36093 {}] regionserver.HRegion(3140): Finished flush of dataSize ~34.67 KB/35506, heapSize ~42.73 KB/43752, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 68ms, sequenceid=84, compaction requested=false 2024-11-26T21:52:08,608 INFO [M:0;828b3f090c91:36093 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-26T21:52:08,608 DEBUG [M:0;828b3f090c91:36093 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1732657928538Disabling compacts and flushes for region at 1732657928538Disabling writes for close at 1732657928538Obtaining lock to block concurrent updates at 1732657928539 (+1 ms)Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1732657928539Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=35506, getHeapSize=43992, getOffHeapSize=0, getCellsCount=99 at 1732657928539Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1732657928539Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1732657928539Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1732657928559 (+20 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1732657928559Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1732657928562 (+3 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1732657928579 (+17 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1732657928579Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1732657928582 (+3 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1732657928598 (+16 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1732657928598Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@77d0bd63: reopening flushed file at 1732657928600 (+2 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@37959cef: reopening flushed file at 1732657928602 (+2 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@13452a60: reopening flushed file at 1732657928604 (+2 ms)Finished flush of dataSize ~34.67 KB/35506, heapSize ~42.73 KB/43752, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 68ms, sequenceid=84, compaction requested=false at 1732657928606 (+2 ms)Writing region close event to WAL at 1732657928608 (+2 ms)Closed at 1732657928608 2024-11-26T21:52:08,610 INFO [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-26T21:52:08,610 INFO [M:0;828b3f090c91:36093 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down. 2024-11-26T21:52:08,610 INFO [M:0;828b3f090c91:36093 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:36093 2024-11-26T21:52:08,611 INFO [M:0;828b3f090c91:36093 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-26T21:52:08,625 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35731-0x1007e051f2e0001, quorum=127.0.0.1:64198, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-26T21:52:08,625 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39637-0x1007e051f2e0002, quorum=127.0.0.1:64198, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-26T21:52:08,625 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39637-0x1007e051f2e0002, quorum=127.0.0.1:64198, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-26T21:52:08,625 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35731-0x1007e051f2e0001, quorum=127.0.0.1:64198, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-26T21:52:08,625 INFO [RS:1;828b3f090c91:39637 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-26T21:52:08,625 INFO [RS:0;828b3f090c91:35731 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-26T21:52:08,625 INFO [RS:1;828b3f090c91:39637 {}] regionserver.HRegionServer(1031): Exiting; stopping=828b3f090c91,39637,1732657923848; zookeeper connection closed. 2024-11-26T21:52:08,625 INFO [RS:0;828b3f090c91:35731 {}] regionserver.HRegionServer(1031): Exiting; stopping=828b3f090c91,35731,1732657923709; zookeeper connection closed. 2024-11-26T21:52:08,626 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@3a8bc00d {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@3a8bc00d 2024-11-26T21:52:08,626 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@105e131b {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@105e131b 2024-11-26T21:52:08,626 INFO [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 2 regionserver(s) complete 2024-11-26T21:52:08,712 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36093-0x1007e051f2e0000, quorum=127.0.0.1:64198, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-26T21:52:08,713 INFO [M:0;828b3f090c91:36093 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-26T21:52:08,713 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36093-0x1007e051f2e0000, quorum=127.0.0.1:64198, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-26T21:52:08,723 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers 2024-11-26T21:52:08,741 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down 2024-11-26T21:52:08,742 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-26T21:52:08,749 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: io.hfile.TestBlockEvictionOnRegionMovement#testBlockEvictionOnRegionMove Thread=50 (was 12) Potentially hanging thread: master/828b3f090c91:0:becomeActiveMaster-MemStoreChunkPool Statistics java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Timer for 'HBase' metrics system java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.util.TimerThread.mainLoop(Timer.java:563) java.base@17.0.11/java.util.TimerThread.run(Timer.java:516) Potentially hanging thread: regionserver/828b3f090c91:0.procedureResultReporter java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.RemoteProcedureResultReporter.run(RemoteProcedureResultReporter.java:75) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-3-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: SnapshotHandlerChoreCleaner java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HMaster-EventLoopGroup-1-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-3-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: hfile-prefetch-1732657926695 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: hfile-prefetch-1732657928389 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1177) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RpcClient-timer-pool-0 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.waitForNextTick(HashedWheelTimer.java:598) app//org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:494) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Idle-Rpc-Conn-Sweeper-pool-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-4-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Monitor thread for TaskMonitor java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hbase.monitoring.TaskMonitor$MonitorRunnable.run(TaskMonitor.java:325) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ForkJoinPool.commonPool-worker-3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.ForkJoinPool.awaitWork(ForkJoinPool.java:1724) java.base@17.0.11/java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1623) java.base@17.0.11/java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:165) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-4-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: weak-ref-cleaner-strictcontextstorage java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:155) java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:176) app//io.opentelemetry.context.StrictContextStorage$PendingScopes.run(StrictContextStorage.java:269) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: hfile-prefetch-1732657928386 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1177) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HBase-Metrics2-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: master/828b3f090c91:0:becomeActiveMaster-MemStoreChunkPool Statistics java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: hfile-prefetch-1732657928243 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1177) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: SessionTracker java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.zookeeper.server.SessionTrackerImpl.run(SessionTrackerImpl.java:163) Potentially hanging thread: Time-limited test.named-queue-events-pool-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) app//com.lmax.disruptor.BlockingWaitStrategy.waitFor(BlockingWaitStrategy.java:47) app//com.lmax.disruptor.ProcessingSequenceBarrier.waitFor(ProcessingSequenceBarrier.java:56) app//com.lmax.disruptor.BatchEventProcessor.processEvents(BatchEventProcessor.java:159) app//com.lmax.disruptor.BatchEventProcessor.run(BatchEventProcessor.java:125) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: regionserver/828b3f090c91:0.procedureResultReporter java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.RemoteProcedureResultReporter.run(RemoteProcedureResultReporter.java:75) Potentially hanging thread: org.apache.hadoop.fs.FileSystem$Statistics$StatisticsDataReferenceCleaner java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:155) java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:176) app//org.apache.hadoop.fs.FileSystem$Statistics$StatisticsDataReferenceCleaner.run(FileSystem.java:4171) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HMaster-EventLoopGroup-1-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: region-location-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1170) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HMaster-EventLoopGroup-1-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-4 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Async-Client-Retry-Timer-pool-0 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.waitForNextTick(HashedWheelTimer.java:598) app//org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:494) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-4-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ForkJoinPool.commonPool-worker-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkUntil(LockSupport.java:410) java.base@17.0.11/java.util.concurrent.ForkJoinPool.awaitWork(ForkJoinPool.java:1726) java.base@17.0.11/java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1623) java.base@17.0.11/java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:165) Potentially hanging thread: region-location-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1170) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ForkJoinPool.commonPool-worker-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.ForkJoinPool.awaitWork(ForkJoinPool.java:1724) java.base@17.0.11/java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1623) java.base@17.0.11/java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:165) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-3-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=346 (was 287) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=137 (was 145), ProcessCount=11 (was 11), AvailableMemoryMB=10931 (was 11286) 2024-11-26T21:52:08,755 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: io.hfile.TestBlockEvictionOnRegionMovement#testBlockEvictionOnGracefulStop Thread=50, OpenFileDescriptor=346, MaxFileDescriptor=1048576, SystemLoadAverage=137, ProcessCount=11, AvailableMemoryMB=10931 2024-11-26T21:52:08,756 INFO [Time-limited test {}] hbase.HBaseTestingUtil(401): System.getProperty("hadoop.log.dir") already set to: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7430f913-2694-54cf-261c-654b69c5fe06/hadoop-log-dir so I do NOT create it in target/test-data/b34d8522-cb7c-4d9b-6a0e-fe88ad6cba6d 2024-11-26T21:52:08,756 INFO [Time-limited test {}] hbase.HBaseTestingUtil(401): System.getProperty("hadoop.tmp.dir") already set to: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7430f913-2694-54cf-261c-654b69c5fe06/hadoop-tmp-dir so I do NOT create it in target/test-data/b34d8522-cb7c-4d9b-6a0e-fe88ad6cba6d 2024-11-26T21:52:08,756 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-26T21:52:08,759 INFO [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/b34d8522-cb7c-4d9b-6a0e-fe88ad6cba6d/cluster_3bce7550-bfd8-d951-5f2a-b8992c9e0b57, deleteOnExit=true 2024-11-26T21:52:08,765 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/b34d8522-cb7c-4d9b-6a0e-fe88ad6cba6d/cluster_3bce7550-bfd8-d951-5f2a-b8992c9e0b57/zookeeper_0, clientPort=53598, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/b34d8522-cb7c-4d9b-6a0e-fe88ad6cba6d/cluster_3bce7550-bfd8-d951-5f2a-b8992c9e0b57/zookeeper_0/version-2, dataDirSize=457 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/b34d8522-cb7c-4d9b-6a0e-fe88ad6cba6d/cluster_3bce7550-bfd8-d951-5f2a-b8992c9e0b57/zookeeper_0/version-2, dataLogSize=457 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0 2024-11-26T21:52:08,767 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=53598 2024-11-26T21:52:08,777 INFO [Time-limited test {}] util.FSUtils(489): Created version file at /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7430f913-2694-54cf-261c-654b69c5fe06/de866367-466a-3788-a91b-60c9e5aa6ab2 with version=8 2024-11-26T21:52:08,777 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1141): The hbase.fs.tmp.dir is set to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7430f913-2694-54cf-261c-654b69c5fe06/de866367-466a-3788-a91b-60c9e5aa6ab2/hbase-staging 2024-11-26T21:52:08,780 INFO [Time-limited test {}] client.ConnectionUtils(128): master/828b3f090c91:0 server-side Connection retries=45 2024-11-26T21:52:08,780 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-26T21:52:08,780 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-26T21:52:08,780 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-26T21:52:08,780 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-26T21:52:08,780 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-26T21:52:08,780 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService 2024-11-26T21:52:08,781 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-26T21:52:08,781 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:35155 2024-11-26T21:52:08,783 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:35155 connecting to ZooKeeper ensemble=127.0.0.1:53598 2024-11-26T21:52:08,788 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:351550x0, quorum=127.0.0.1:53598, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-26T21:52:08,788 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:35155-0x1007e0536a90000 connected 2024-11-26T21:52:08,804 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-26T21:52:08,805 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-26T21:52:08,806 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:35155-0x1007e0536a90000, quorum=127.0.0.1:53598, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-26T21:52:08,807 INFO [Time-limited test {}] master.HMaster(525): hbase.rootdir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7430f913-2694-54cf-261c-654b69c5fe06/de866367-466a-3788-a91b-60c9e5aa6ab2, hbase.cluster.distributed=false 2024-11-26T21:52:08,809 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:35155-0x1007e0536a90000, quorum=127.0.0.1:53598, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-26T21:52:08,810 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=35155 2024-11-26T21:52:08,811 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=35155 2024-11-26T21:52:08,811 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=35155 2024-11-26T21:52:08,812 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=35155 2024-11-26T21:52:08,812 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=35155 2024-11-26T21:52:08,830 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/828b3f090c91:0 server-side Connection retries=45 2024-11-26T21:52:08,830 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-26T21:52:08,830 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-26T21:52:08,830 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-26T21:52:08,830 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-26T21:52:08,830 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-26T21:52:08,830 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-26T21:52:08,830 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-26T21:52:08,831 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:33857 2024-11-26T21:52:08,833 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:33857 connecting to ZooKeeper ensemble=127.0.0.1:53598 2024-11-26T21:52:08,833 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-26T21:52:08,833 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-26T21:52:08,835 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:338570x0, quorum=127.0.0.1:53598, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-26T21:52:08,836 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:33857-0x1007e0536a90001 connected 2024-11-26T21:52:08,836 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:33857-0x1007e0536a90001, quorum=127.0.0.1:53598, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-26T21:52:08,836 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-26T21:52:08,837 INFO [Time-limited test {}] bucket.FileIOEngine(99): Allocating cache 400 MB, on the path:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/b34d8522-cb7c-4d9b-6a0e-fe88ad6cba6d/bucket.cache 2024-11-26T21:52:08,837 INFO [Time-limited test {}] bucket.BucketCache(364): Instantiating BucketCache with acceptableFactor: 0.95, minFactor: 0.85, extraFreeFactor: 0.1, singleFactor: 0.25, multiFactor: 0.5, memoryFactor: 0.25, useStrongRef: false 2024-11-26T21:52:08,837 INFO [Time-limited test {}] bucket.BucketCache(478): Starting BucketCachePersisterThread 2024-11-26T21:52:08,837 INFO [Time-limited test {}] bucket.BucketCachePersister(38): BucketCachePersister started with interval: 100 2024-11-26T21:52:08,838 INFO [Thread-118 {}] bucket.BucketCache(1530): Started retrieving bucket cache from file 2024-11-26T21:52:08,838 WARN [Thread-118 {}] bucket.BucketCache(1533): Persistence file missing! It's ok if it's first run after enabling persistent cache. 2024-11-26T21:52:08,838 INFO [Time-limited test {}] bucket.BucketCache(403): Started bucket cache; ioengine=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/b34d8522-cb7c-4d9b-6a0e-fe88ad6cba6d/bucket.cache, capacity=400 MB, blockSize=64 KB, writerThreadNum=3, writerQLen=64, persistencePath=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/b34d8522-cb7c-4d9b-6a0e-fe88ad6cba6d/bucket.persistence, bucketAllocator=org.apache.hadoop.hbase.io.hfile.bucket.BucketAllocator 2024-11-26T21:52:08,838 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-26T21:52:08,838 INFO [Thread-118 {}] bucket.BucketAllocator(343): Cache totalSize=418148352, buckets=199, bucket capacity=2101248=(4*525312)=(FEWEST_ITEMS_IN_BUCKET*(largest configured bucketcache size)) 2024-11-26T21:52:08,838 INFO [Thread-118 {}] bucket.BucketCache(413): Persistent bucket cache recovery from /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/b34d8522-cb7c-4d9b-6a0e-fe88ad6cba6d/bucket.persistence is complete. 2024-11-26T21:52:08,839 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:33857-0x1007e0536a90001, quorum=127.0.0.1:53598, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-26T21:52:08,840 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:33857-0x1007e0536a90001, quorum=127.0.0.1:53598, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-26T21:52:08,841 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=33857 2024-11-26T21:52:08,841 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=33857 2024-11-26T21:52:08,841 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=33857 2024-11-26T21:52:08,842 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=33857 2024-11-26T21:52:08,842 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=33857 2024-11-26T21:52:08,859 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/828b3f090c91:0 server-side Connection retries=45 2024-11-26T21:52:08,859 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-26T21:52:08,859 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-26T21:52:08,859 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-26T21:52:08,859 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-26T21:52:08,859 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-26T21:52:08,859 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-26T21:52:08,860 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-26T21:52:08,860 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:42409 2024-11-26T21:52:08,862 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:42409 connecting to ZooKeeper ensemble=127.0.0.1:53598 2024-11-26T21:52:08,862 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-26T21:52:08,862 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-26T21:52:08,865 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:424090x0, quorum=127.0.0.1:53598, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-26T21:52:08,865 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:424090x0, quorum=127.0.0.1:53598, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-26T21:52:08,865 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:42409-0x1007e0536a90002 connected 2024-11-26T21:52:08,866 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-26T21:52:08,866 INFO [Time-limited test {}] bucket.FileIOEngine(99): Allocating cache 400 MB, on the path:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/b34d8522-cb7c-4d9b-6a0e-fe88ad6cba6d/bucket.cache 2024-11-26T21:52:08,866 INFO [Time-limited test {}] bucket.BucketCache(364): Instantiating BucketCache with acceptableFactor: 0.95, minFactor: 0.85, extraFreeFactor: 0.1, singleFactor: 0.25, multiFactor: 0.5, memoryFactor: 0.25, useStrongRef: false 2024-11-26T21:52:08,867 INFO [Time-limited test {}] bucket.BucketCache(478): Starting BucketCachePersisterThread 2024-11-26T21:52:08,867 INFO [Time-limited test {}] bucket.BucketCachePersister(38): BucketCachePersister started with interval: 100 2024-11-26T21:52:08,867 INFO [Thread-119 {}] bucket.BucketCache(1530): Started retrieving bucket cache from file 2024-11-26T21:52:08,867 WARN [Thread-119 {}] bucket.BucketCache(1533): Persistence file missing! It's ok if it's first run after enabling persistent cache. 2024-11-26T21:52:08,867 INFO [Time-limited test {}] bucket.BucketCache(403): Started bucket cache; ioengine=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/b34d8522-cb7c-4d9b-6a0e-fe88ad6cba6d/bucket.cache, capacity=400 MB, blockSize=64 KB, writerThreadNum=3, writerQLen=64, persistencePath=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/b34d8522-cb7c-4d9b-6a0e-fe88ad6cba6d/bucket.persistence, bucketAllocator=org.apache.hadoop.hbase.io.hfile.bucket.BucketAllocator 2024-11-26T21:52:08,867 INFO [Thread-119 {}] bucket.BucketAllocator(343): Cache totalSize=418148352, buckets=199, bucket capacity=2101248=(4*525312)=(FEWEST_ITEMS_IN_BUCKET*(largest configured bucketcache size)) 2024-11-26T21:52:08,867 INFO [Thread-119 {}] bucket.BucketCache(413): Persistent bucket cache recovery from /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/b34d8522-cb7c-4d9b-6a0e-fe88ad6cba6d/bucket.persistence is complete. 2024-11-26T21:52:08,867 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-26T21:52:08,868 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:42409-0x1007e0536a90002, quorum=127.0.0.1:53598, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-26T21:52:08,870 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:42409-0x1007e0536a90002, quorum=127.0.0.1:53598, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-26T21:52:08,870 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=42409 2024-11-26T21:52:08,870 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=42409 2024-11-26T21:52:08,871 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=42409 2024-11-26T21:52:08,871 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=42409 2024-11-26T21:52:08,871 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=42409 2024-11-26T21:52:08,883 DEBUG [M:0;828b3f090c91:35155 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;828b3f090c91:35155 2024-11-26T21:52:08,883 INFO [master/828b3f090c91:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/828b3f090c91,35155,1732657928779 2024-11-26T21:52:08,884 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42409-0x1007e0536a90002, quorum=127.0.0.1:53598, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-26T21:52:08,884 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35155-0x1007e0536a90000, quorum=127.0.0.1:53598, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-26T21:52:08,884 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33857-0x1007e0536a90001, quorum=127.0.0.1:53598, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-26T21:52:08,885 DEBUG [master/828b3f090c91:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:35155-0x1007e0536a90000, quorum=127.0.0.1:53598, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/828b3f090c91,35155,1732657928779 2024-11-26T21:52:08,886 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33857-0x1007e0536a90001, quorum=127.0.0.1:53598, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-26T21:52:08,886 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42409-0x1007e0536a90002, quorum=127.0.0.1:53598, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-26T21:52:08,886 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33857-0x1007e0536a90001, quorum=127.0.0.1:53598, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-26T21:52:08,886 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42409-0x1007e0536a90002, quorum=127.0.0.1:53598, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-26T21:52:08,887 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35155-0x1007e0536a90000, quorum=127.0.0.1:53598, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-26T21:52:08,887 DEBUG [master/828b3f090c91:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:35155-0x1007e0536a90000, quorum=127.0.0.1:53598, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-11-26T21:52:08,888 INFO [master/828b3f090c91:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/828b3f090c91,35155,1732657928779 from backup master directory 2024-11-26T21:52:08,889 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35155-0x1007e0536a90000, quorum=127.0.0.1:53598, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/828b3f090c91,35155,1732657928779 2024-11-26T21:52:08,889 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33857-0x1007e0536a90001, quorum=127.0.0.1:53598, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-26T21:52:08,889 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42409-0x1007e0536a90002, quorum=127.0.0.1:53598, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-26T21:52:08,889 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35155-0x1007e0536a90000, quorum=127.0.0.1:53598, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-26T21:52:08,889 WARN [master/828b3f090c91:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-26T21:52:08,889 INFO [master/828b3f090c91:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=828b3f090c91,35155,1732657928779 2024-11-26T21:52:08,890 DEBUG [master/828b3f090c91:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7430f913-2694-54cf-261c-654b69c5fe06/de866367-466a-3788-a91b-60c9e5aa6ab2/hbase.id] with ID: 45b5085c-0c14-4246-aa10-be0654098e00 2024-11-26T21:52:08,890 DEBUG [master/828b3f090c91:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7430f913-2694-54cf-261c-654b69c5fe06/de866367-466a-3788-a91b-60c9e5aa6ab2/.tmp/hbase.id 2024-11-26T21:52:08,897 DEBUG [master/828b3f090c91:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7430f913-2694-54cf-261c-654b69c5fe06/de866367-466a-3788-a91b-60c9e5aa6ab2/.tmp/hbase.id]:[file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7430f913-2694-54cf-261c-654b69c5fe06/de866367-466a-3788-a91b-60c9e5aa6ab2/hbase.id] 2024-11-26T21:52:08,925 DEBUG [master/828b3f090c91:0:becomeActiveMaster {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-26T21:52:08,925 INFO [master/828b3f090c91:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem. 2024-11-26T21:52:08,928 INFO [master/828b3f090c91:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 3ms. 2024-11-26T21:52:08,930 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42409-0x1007e0536a90002, quorum=127.0.0.1:53598, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-26T21:52:08,930 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33857-0x1007e0536a90001, quorum=127.0.0.1:53598, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-26T21:52:08,930 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35155-0x1007e0536a90000, quorum=127.0.0.1:53598, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-26T21:52:08,935 INFO [master/828b3f090c91:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-26T21:52:08,936 INFO [master/828b3f090c91:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-11-26T21:52:08,936 INFO [master/828b3f090c91:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-26T21:52:08,954 INFO [master/828b3f090c91:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7430f913-2694-54cf-261c-654b69c5fe06/de866367-466a-3788-a91b-60c9e5aa6ab2/MasterData/data/master/store 2024-11-26T21:52:08,960 DEBUG [master/828b3f090c91:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-26T21:52:08,960 DEBUG [master/828b3f090c91:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-26T21:52:08,960 INFO [master/828b3f090c91:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-26T21:52:08,960 DEBUG [master/828b3f090c91:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-26T21:52:08,961 DEBUG [master/828b3f090c91:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-26T21:52:08,961 DEBUG [master/828b3f090c91:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-26T21:52:08,961 INFO [master/828b3f090c91:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-26T21:52:08,961 DEBUG [master/828b3f090c91:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1732657928960Disabling compacts and flushes for region at 1732657928960Disabling writes for close at 1732657928961 (+1 ms)Writing region close event to WAL at 1732657928961Closed at 1732657928961 2024-11-26T21:52:08,961 WARN [master/828b3f090c91:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7430f913-2694-54cf-261c-654b69c5fe06/de866367-466a-3788-a91b-60c9e5aa6ab2/MasterData/data/master/store/.initializing 2024-11-26T21:52:08,961 DEBUG [master/828b3f090c91:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7430f913-2694-54cf-261c-654b69c5fe06/de866367-466a-3788-a91b-60c9e5aa6ab2/MasterData/WALs/828b3f090c91,35155,1732657928779 2024-11-26T21:52:08,966 INFO [master/828b3f090c91:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-26T21:52:08,969 INFO [master/828b3f090c91:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=828b3f090c91%2C35155%2C1732657928779, suffix=, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7430f913-2694-54cf-261c-654b69c5fe06/de866367-466a-3788-a91b-60c9e5aa6ab2/MasterData/WALs/828b3f090c91,35155,1732657928779, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7430f913-2694-54cf-261c-654b69c5fe06/de866367-466a-3788-a91b-60c9e5aa6ab2/MasterData/oldWALs, maxLogs=10 2024-11-26T21:52:08,986 INFO [master/828b3f090c91:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7430f913-2694-54cf-261c-654b69c5fe06/de866367-466a-3788-a91b-60c9e5aa6ab2/MasterData/WALs/828b3f090c91,35155,1732657928779/828b3f090c91%2C35155%2C1732657928779.1732657928970 2024-11-26T21:52:08,986 DEBUG [master/828b3f090c91:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-11-26T21:52:08,986 DEBUG [master/828b3f090c91:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-11-26T21:52:08,986 DEBUG [master/828b3f090c91:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-26T21:52:08,986 DEBUG [master/828b3f090c91:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-11-26T21:52:08,986 DEBUG [master/828b3f090c91:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-11-26T21:52:08,987 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-11-26T21:52:08,992 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-11-26T21:52:08,992 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T21:52:08,992 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-26T21:52:08,993 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-11-26T21:52:08,996 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-11-26T21:52:08,996 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T21:52:08,996 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-26T21:52:08,997 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-11-26T21:52:09,000 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-11-26T21:52:09,000 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T21:52:09,000 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-26T21:52:09,001 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-11-26T21:52:09,004 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-11-26T21:52:09,005 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T21:52:09,005 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-26T21:52:09,005 DEBUG [master/828b3f090c91:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682 2024-11-26T21:52:09,006 DEBUG [master/828b3f090c91:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7430f913-2694-54cf-261c-654b69c5fe06/de866367-466a-3788-a91b-60c9e5aa6ab2/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-26T21:52:09,006 DEBUG [master/828b3f090c91:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7430f913-2694-54cf-261c-654b69c5fe06/de866367-466a-3788-a91b-60c9e5aa6ab2/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-26T21:52:09,006 DEBUG [master/828b3f090c91:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682 2024-11-26T21:52:09,006 DEBUG [master/828b3f090c91:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682 2024-11-26T21:52:09,006 DEBUG [master/828b3f090c91:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-26T21:52:09,007 DEBUG [master/828b3f090c91:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-11-26T21:52:09,013 DEBUG [master/828b3f090c91:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7430f913-2694-54cf-261c-654b69c5fe06/de866367-466a-3788-a91b-60c9e5aa6ab2/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-26T21:52:09,013 INFO [master/828b3f090c91:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=60692032, jitterRate=-0.09561824798583984}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-26T21:52:09,014 DEBUG [master/828b3f090c91:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682: Writing region info on filesystem at 1732657928986Initializing all the Stores at 1732657928987 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732657928987Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732657928987Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732657928987Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732657928987Cleaning up temporary data from old regions at 1732657929006 (+19 ms)Region opened successfully at 1732657929014 (+8 ms) 2024-11-26T21:52:09,014 INFO [master/828b3f090c91:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-11-26T21:52:09,018 DEBUG [master/828b3f090c91:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@453dd31b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=828b3f090c91/172.17.0.2:0 2024-11-26T21:52:09,019 INFO [master/828b3f090c91:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating... 2024-11-26T21:52:09,019 INFO [master/828b3f090c91:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5 2024-11-26T21:52:09,019 INFO [master/828b3f090c91:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50 2024-11-26T21:52:09,019 INFO [master/828b3f090c91:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-26T21:52:09,020 INFO [master/828b3f090c91:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-26T21:52:09,020 INFO [master/828b3f090c91:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 0 msec 2024-11-26T21:52:09,020 INFO [master/828b3f090c91:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-11-26T21:52:09,022 INFO [master/828b3f090c91:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2024-11-26T21:52:09,023 DEBUG [master/828b3f090c91:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:35155-0x1007e0536a90000, quorum=127.0.0.1:53598, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error) 2024-11-26T21:52:09,024 DEBUG [master/828b3f090c91:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false 2024-11-26T21:52:09,025 INFO [master/828b3f090c91:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2024-11-26T21:52:09,026 DEBUG [master/828b3f090c91:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:35155-0x1007e0536a90000, quorum=127.0.0.1:53598, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error) 2024-11-26T21:52:09,027 DEBUG [master/828b3f090c91:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false 2024-11-26T21:52:09,027 INFO [master/828b3f090c91:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2024-11-26T21:52:09,028 DEBUG [master/828b3f090c91:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:35155-0x1007e0536a90000, quorum=127.0.0.1:53598, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error) 2024-11-26T21:52:09,029 DEBUG [master/828b3f090c91:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false 2024-11-26T21:52:09,030 DEBUG [master/828b3f090c91:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:35155-0x1007e0536a90000, quorum=127.0.0.1:53598, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error) 2024-11-26T21:52:09,031 DEBUG [master/828b3f090c91:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false 2024-11-26T21:52:09,033 DEBUG [master/828b3f090c91:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:35155-0x1007e0536a90000, quorum=127.0.0.1:53598, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error) 2024-11-26T21:52:09,034 DEBUG [master/828b3f090c91:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false 2024-11-26T21:52:09,036 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42409-0x1007e0536a90002, quorum=127.0.0.1:53598, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-26T21:52:09,036 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33857-0x1007e0536a90001, quorum=127.0.0.1:53598, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-26T21:52:09,036 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42409-0x1007e0536a90002, quorum=127.0.0.1:53598, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-26T21:52:09,036 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33857-0x1007e0536a90001, quorum=127.0.0.1:53598, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-26T21:52:09,036 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35155-0x1007e0536a90000, quorum=127.0.0.1:53598, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-26T21:52:09,036 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35155-0x1007e0536a90000, quorum=127.0.0.1:53598, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-26T21:52:09,037 INFO [master/828b3f090c91:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=828b3f090c91,35155,1732657928779, sessionid=0x1007e0536a90000, setting cluster-up flag (Was=false) 2024-11-26T21:52:09,045 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33857-0x1007e0536a90001, quorum=127.0.0.1:53598, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-26T21:52:09,045 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42409-0x1007e0536a90002, quorum=127.0.0.1:53598, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-26T21:52:09,045 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35155-0x1007e0536a90000, quorum=127.0.0.1:53598, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-26T21:52:09,049 DEBUG [master/828b3f090c91:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort 2024-11-26T21:52:09,050 DEBUG [master/828b3f090c91:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=828b3f090c91,35155,1732657928779 2024-11-26T21:52:09,052 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42409-0x1007e0536a90002, quorum=127.0.0.1:53598, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-26T21:52:09,052 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33857-0x1007e0536a90001, quorum=127.0.0.1:53598, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-26T21:52:09,052 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35155-0x1007e0536a90000, quorum=127.0.0.1:53598, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-26T21:52:09,058 DEBUG [master/828b3f090c91:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort 2024-11-26T21:52:09,059 DEBUG [master/828b3f090c91:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=828b3f090c91,35155,1732657928779 2024-11-26T21:52:09,060 INFO [master/828b3f090c91:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7430f913-2694-54cf-261c-654b69c5fe06/de866367-466a-3788-a91b-60c9e5aa6ab2/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again 2024-11-26T21:52:09,062 DEBUG [master/828b3f090c91:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-26T21:52:09,062 INFO [master/828b3f090c91:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2 2024-11-26T21:52:09,062 INFO [master/828b3f090c91:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2024-11-26T21:52:09,063 DEBUG [master/828b3f090c91:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: 828b3f090c91,35155,1732657928779 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2024-11-26T21:52:09,064 DEBUG [master/828b3f090c91:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/828b3f090c91:0, corePoolSize=5, maxPoolSize=5 2024-11-26T21:52:09,064 DEBUG [master/828b3f090c91:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/828b3f090c91:0, corePoolSize=5, maxPoolSize=5 2024-11-26T21:52:09,064 DEBUG [master/828b3f090c91:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/828b3f090c91:0, corePoolSize=5, maxPoolSize=5 2024-11-26T21:52:09,065 DEBUG [master/828b3f090c91:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/828b3f090c91:0, corePoolSize=5, maxPoolSize=5 2024-11-26T21:52:09,065 DEBUG [master/828b3f090c91:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/828b3f090c91:0, corePoolSize=10, maxPoolSize=10 2024-11-26T21:52:09,065 DEBUG [master/828b3f090c91:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/828b3f090c91:0, corePoolSize=1, maxPoolSize=1 2024-11-26T21:52:09,065 DEBUG [master/828b3f090c91:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/828b3f090c91:0, corePoolSize=2, maxPoolSize=2 2024-11-26T21:52:09,065 DEBUG [master/828b3f090c91:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/828b3f090c91:0, corePoolSize=1, maxPoolSize=1 2024-11-26T21:52:09,066 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-26T21:52:09,067 INFO [master/828b3f090c91:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732657959066 2024-11-26T21:52:09,067 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region 2024-11-26T21:52:09,067 INFO [master/828b3f090c91:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2024-11-26T21:52:09,067 INFO [master/828b3f090c91:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2024-11-26T21:52:09,067 INFO [master/828b3f090c91:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2024-11-26T21:52:09,067 INFO [master/828b3f090c91:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2024-11-26T21:52:09,067 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T21:52:09,067 INFO [master/828b3f090c91:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2024-11-26T21:52:09,067 INFO [master/828b3f090c91:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2024-11-26T21:52:09,067 INFO [master/828b3f090c91:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-26T21:52:09,067 INFO [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-11-26T21:52:09,068 INFO [master/828b3f090c91:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2024-11-26T21:52:09,068 INFO [master/828b3f090c91:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2024-11-26T21:52:09,068 INFO [master/828b3f090c91:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2024-11-26T21:52:09,068 INFO [master/828b3f090c91:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2024-11-26T21:52:09,068 INFO [master/828b3f090c91:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2024-11-26T21:52:09,069 DEBUG [master/828b3f090c91:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/828b3f090c91:0:becomeActiveMaster-HFileCleaner.large.0-1732657929069,5,FailOnTimeoutGroup] 2024-11-26T21:52:09,069 DEBUG [master/828b3f090c91:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/828b3f090c91:0:becomeActiveMaster-HFileCleaner.small.0-1732657929069,5,FailOnTimeoutGroup] 2024-11-26T21:52:09,069 INFO [master/828b3f090c91:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-26T21:52:09,069 INFO [master/828b3f090c91:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2024-11-26T21:52:09,069 INFO [master/828b3f090c91:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2024-11-26T21:52:09,069 INFO [master/828b3f090c91:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2024-11-26T21:52:09,074 INFO [RS:1;828b3f090c91:42409 {}] regionserver.HRegionServer(746): ClusterId : 45b5085c-0c14-4246-aa10-be0654098e00 2024-11-26T21:52:09,074 INFO [RS:0;828b3f090c91:33857 {}] regionserver.HRegionServer(746): ClusterId : 45b5085c-0c14-4246-aa10-be0654098e00 2024-11-26T21:52:09,075 DEBUG [RS:0;828b3f090c91:33857 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-26T21:52:09,075 DEBUG [RS:1;828b3f090c91:42409 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-26T21:52:09,078 DEBUG [RS:0;828b3f090c91:33857 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-26T21:52:09,078 DEBUG [RS:1;828b3f090c91:42409 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-26T21:52:09,078 DEBUG [RS:1;828b3f090c91:42409 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-26T21:52:09,078 DEBUG [RS:0;828b3f090c91:33857 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-26T21:52:09,080 DEBUG [RS:0;828b3f090c91:33857 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-26T21:52:09,080 DEBUG [RS:1;828b3f090c91:42409 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-26T21:52:09,080 DEBUG [RS:1;828b3f090c91:42409 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@45bb8079, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=828b3f090c91/172.17.0.2:0 2024-11-26T21:52:09,081 DEBUG [RS:0;828b3f090c91:33857 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@bff134e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=828b3f090c91/172.17.0.2:0 2024-11-26T21:52:09,098 DEBUG [RS:0;828b3f090c91:33857 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;828b3f090c91:33857 2024-11-26T21:52:09,098 INFO [RS:0;828b3f090c91:33857 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-26T21:52:09,098 INFO [RS:0;828b3f090c91:33857 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-26T21:52:09,098 DEBUG [RS:0;828b3f090c91:33857 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-26T21:52:09,098 INFO [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7430f913-2694-54cf-261c-654b69c5fe06/de866367-466a-3788-a91b-60c9e5aa6ab2/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321 2024-11-26T21:52:09,099 DEBUG [RS:1;828b3f090c91:42409 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:1;828b3f090c91:42409 2024-11-26T21:52:09,099 INFO [RS:1;828b3f090c91:42409 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-26T21:52:09,099 INFO [RS:1;828b3f090c91:42409 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-26T21:52:09,099 DEBUG [RS:1;828b3f090c91:42409 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-26T21:52:09,099 INFO [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7430f913-2694-54cf-261c-654b69c5fe06/de866367-466a-3788-a91b-60c9e5aa6ab2 2024-11-26T21:52:09,099 INFO [RS:0;828b3f090c91:33857 {}] regionserver.HRegionServer(2659): reportForDuty to master=828b3f090c91,35155,1732657928779 with port=33857, startcode=1732657928830 2024-11-26T21:52:09,099 DEBUG [RS:0;828b3f090c91:33857 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-26T21:52:09,100 INFO [RS:1;828b3f090c91:42409 {}] regionserver.HRegionServer(2659): reportForDuty to master=828b3f090c91,35155,1732657928779 with port=42409, startcode=1732657928858 2024-11-26T21:52:09,100 DEBUG [RS:1;828b3f090c91:42409 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-26T21:52:09,106 INFO [HMaster-EventLoopGroup-6-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50923, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=RegionServerStatusService 2024-11-26T21:52:09,107 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=35155 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 828b3f090c91,42409,1732657928858 2024-11-26T21:52:09,107 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=35155 {}] master.ServerManager(517): Registering regionserver=828b3f090c91,42409,1732657928858 2024-11-26T21:52:09,107 INFO [HMaster-EventLoopGroup-6-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52697, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=RegionServerStatusService 2024-11-26T21:52:09,109 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=35155 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 828b3f090c91,33857,1732657928830 2024-11-26T21:52:09,109 DEBUG [RS:1;828b3f090c91:42409 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7430f913-2694-54cf-261c-654b69c5fe06/de866367-466a-3788-a91b-60c9e5aa6ab2 2024-11-26T21:52:09,109 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=35155 {}] master.ServerManager(517): Registering regionserver=828b3f090c91,33857,1732657928830 2024-11-26T21:52:09,109 DEBUG [RS:1;828b3f090c91:42409 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=file:/ 2024-11-26T21:52:09,109 DEBUG [RS:1;828b3f090c91:42409 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-26T21:52:09,111 DEBUG [RS:0;828b3f090c91:33857 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7430f913-2694-54cf-261c-654b69c5fe06/de866367-466a-3788-a91b-60c9e5aa6ab2 2024-11-26T21:52:09,111 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35155-0x1007e0536a90000, quorum=127.0.0.1:53598, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-26T21:52:09,111 DEBUG [RS:0;828b3f090c91:33857 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=file:/ 2024-11-26T21:52:09,111 DEBUG [RS:0;828b3f090c91:33857 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-26T21:52:09,111 DEBUG [RS:1;828b3f090c91:42409 {}] zookeeper.ZKUtil(111): regionserver:42409-0x1007e0536a90002, quorum=127.0.0.1:53598, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/828b3f090c91,42409,1732657928858 2024-11-26T21:52:09,111 WARN [RS:1;828b3f090c91:42409 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-26T21:52:09,112 INFO [RS:1;828b3f090c91:42409 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-26T21:52:09,112 DEBUG [RS:1;828b3f090c91:42409 {}] regionserver.HRegionServer(1793): logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7430f913-2694-54cf-261c-654b69c5fe06/de866367-466a-3788-a91b-60c9e5aa6ab2/WALs/828b3f090c91,42409,1732657928858 2024-11-26T21:52:09,114 DEBUG [RS:0;828b3f090c91:33857 {}] zookeeper.ZKUtil(111): regionserver:33857-0x1007e0536a90001, quorum=127.0.0.1:53598, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/828b3f090c91,33857,1732657928830 2024-11-26T21:52:09,114 WARN [RS:0;828b3f090c91:33857 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-26T21:52:09,114 INFO [RS:0;828b3f090c91:33857 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-26T21:52:09,114 DEBUG [RS:0;828b3f090c91:33857 {}] regionserver.HRegionServer(1793): logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7430f913-2694-54cf-261c-654b69c5fe06/de866367-466a-3788-a91b-60c9e5aa6ab2/WALs/828b3f090c91,33857,1732657928830 2024-11-26T21:52:09,118 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-26T21:52:09,118 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [828b3f090c91,42409,1732657928858] 2024-11-26T21:52:09,118 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [828b3f090c91,33857,1732657928830] 2024-11-26T21:52:09,123 INFO [RS:0;828b3f090c91:33857 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-26T21:52:09,123 INFO [RS:1;828b3f090c91:42409 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-26T21:52:09,129 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family info of region 1588230740 2024-11-26T21:52:09,137 INFO [RS:1;828b3f090c91:42409 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-26T21:52:09,137 INFO [RS:0;828b3f090c91:33857 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-26T21:52:09,138 INFO [RS:0;828b3f090c91:33857 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-26T21:52:09,138 INFO [RS:1;828b3f090c91:42409 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-26T21:52:09,138 INFO [RS:0;828b3f090c91:33857 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-26T21:52:09,138 INFO [RS:1;828b3f090c91:42409 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-26T21:52:09,140 INFO [RS:0;828b3f090c91:33857 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-26T21:52:09,140 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-26T21:52:09,140 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T21:52:09,140 INFO [RS:0;828b3f090c91:33857 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-26T21:52:09,140 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-26T21:52:09,140 INFO [RS:0;828b3f090c91:33857 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-26T21:52:09,140 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family ns of region 1588230740 2024-11-26T21:52:09,141 DEBUG [RS:0;828b3f090c91:33857 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/828b3f090c91:0, corePoolSize=1, maxPoolSize=1 2024-11-26T21:52:09,141 DEBUG [RS:0;828b3f090c91:33857 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/828b3f090c91:0, corePoolSize=1, maxPoolSize=1 2024-11-26T21:52:09,141 DEBUG [RS:0;828b3f090c91:33857 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/828b3f090c91:0, corePoolSize=1, maxPoolSize=1 2024-11-26T21:52:09,141 DEBUG [RS:0;828b3f090c91:33857 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/828b3f090c91:0, corePoolSize=1, maxPoolSize=1 2024-11-26T21:52:09,141 DEBUG [RS:0;828b3f090c91:33857 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/828b3f090c91:0, corePoolSize=1, maxPoolSize=1 2024-11-26T21:52:09,141 DEBUG [RS:0;828b3f090c91:33857 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/828b3f090c91:0, corePoolSize=2, maxPoolSize=2 2024-11-26T21:52:09,141 DEBUG [RS:0;828b3f090c91:33857 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/828b3f090c91:0, corePoolSize=1, maxPoolSize=1 2024-11-26T21:52:09,141 DEBUG [RS:0;828b3f090c91:33857 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/828b3f090c91:0, corePoolSize=1, maxPoolSize=1 2024-11-26T21:52:09,141 DEBUG [RS:0;828b3f090c91:33857 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/828b3f090c91:0, corePoolSize=1, maxPoolSize=1 2024-11-26T21:52:09,141 DEBUG [RS:0;828b3f090c91:33857 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/828b3f090c91:0, corePoolSize=1, maxPoolSize=1 2024-11-26T21:52:09,142 DEBUG [RS:0;828b3f090c91:33857 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/828b3f090c91:0, corePoolSize=1, maxPoolSize=1 2024-11-26T21:52:09,142 DEBUG [RS:0;828b3f090c91:33857 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/828b3f090c91:0, corePoolSize=1, maxPoolSize=1 2024-11-26T21:52:09,142 DEBUG [RS:0;828b3f090c91:33857 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/828b3f090c91:0, corePoolSize=3, maxPoolSize=3 2024-11-26T21:52:09,142 DEBUG [RS:0;828b3f090c91:33857 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/828b3f090c91:0, corePoolSize=3, maxPoolSize=3 2024-11-26T21:52:09,144 INFO [RS:1;828b3f090c91:42409 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-26T21:52:09,145 INFO [RS:1;828b3f090c91:42409 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-26T21:52:09,145 INFO [RS:1;828b3f090c91:42409 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-26T21:52:09,145 DEBUG [RS:1;828b3f090c91:42409 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/828b3f090c91:0, corePoolSize=1, maxPoolSize=1 2024-11-26T21:52:09,146 DEBUG [RS:1;828b3f090c91:42409 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/828b3f090c91:0, corePoolSize=1, maxPoolSize=1 2024-11-26T21:52:09,146 DEBUG [RS:1;828b3f090c91:42409 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/828b3f090c91:0, corePoolSize=1, maxPoolSize=1 2024-11-26T21:52:09,146 DEBUG [RS:1;828b3f090c91:42409 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/828b3f090c91:0, corePoolSize=1, maxPoolSize=1 2024-11-26T21:52:09,147 DEBUG [RS:1;828b3f090c91:42409 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/828b3f090c91:0, corePoolSize=1, maxPoolSize=1 2024-11-26T21:52:09,147 DEBUG [RS:1;828b3f090c91:42409 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/828b3f090c91:0, corePoolSize=2, maxPoolSize=2 2024-11-26T21:52:09,147 DEBUG [RS:1;828b3f090c91:42409 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/828b3f090c91:0, corePoolSize=1, maxPoolSize=1 2024-11-26T21:52:09,147 DEBUG [RS:1;828b3f090c91:42409 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/828b3f090c91:0, corePoolSize=1, maxPoolSize=1 2024-11-26T21:52:09,147 DEBUG [RS:1;828b3f090c91:42409 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/828b3f090c91:0, corePoolSize=1, maxPoolSize=1 2024-11-26T21:52:09,147 DEBUG [RS:1;828b3f090c91:42409 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/828b3f090c91:0, corePoolSize=1, maxPoolSize=1 2024-11-26T21:52:09,147 INFO [RS:0;828b3f090c91:33857 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-26T21:52:09,147 DEBUG [RS:1;828b3f090c91:42409 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/828b3f090c91:0, corePoolSize=1, maxPoolSize=1 2024-11-26T21:52:09,147 INFO [RS:0;828b3f090c91:33857 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-26T21:52:09,147 DEBUG [RS:1;828b3f090c91:42409 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/828b3f090c91:0, corePoolSize=1, maxPoolSize=1 2024-11-26T21:52:09,148 INFO [RS:0;828b3f090c91:33857 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-26T21:52:09,148 DEBUG [RS:1;828b3f090c91:42409 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/828b3f090c91:0, corePoolSize=3, maxPoolSize=3 2024-11-26T21:52:09,148 INFO [RS:0;828b3f090c91:33857 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-26T21:52:09,148 INFO [RS:0;828b3f090c91:33857 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-26T21:52:09,148 DEBUG [RS:1;828b3f090c91:42409 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/828b3f090c91:0, corePoolSize=3, maxPoolSize=3 2024-11-26T21:52:09,148 INFO [RS:0;828b3f090c91:33857 {}] hbase.ChoreService(168): Chore ScheduledChore name=828b3f090c91,33857,1732657928830-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-26T21:52:09,150 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-26T21:52:09,150 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T21:52:09,150 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-26T21:52:09,150 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family rep_barrier of region 1588230740 2024-11-26T21:52:09,154 INFO [RS:1;828b3f090c91:42409 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-26T21:52:09,154 INFO [RS:1;828b3f090c91:42409 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-26T21:52:09,154 INFO [RS:1;828b3f090c91:42409 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-26T21:52:09,154 INFO [RS:1;828b3f090c91:42409 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-26T21:52:09,154 INFO [RS:1;828b3f090c91:42409 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-26T21:52:09,154 INFO [RS:1;828b3f090c91:42409 {}] hbase.ChoreService(168): Chore ScheduledChore name=828b3f090c91,42409,1732657928858-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-26T21:52:09,154 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-26T21:52:09,155 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T21:52:09,155 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-26T21:52:09,155 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family table of region 1588230740 2024-11-26T21:52:09,162 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-26T21:52:09,163 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T21:52:09,163 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-26T21:52:09,163 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-26T21:52:09,164 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7430f913-2694-54cf-261c-654b69c5fe06/de866367-466a-3788-a91b-60c9e5aa6ab2/data/hbase/meta/1588230740 2024-11-26T21:52:09,164 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7430f913-2694-54cf-261c-654b69c5fe06/de866367-466a-3788-a91b-60c9e5aa6ab2/data/hbase/meta/1588230740 2024-11-26T21:52:09,165 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-26T21:52:09,165 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-26T21:52:09,165 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-26T21:52:09,166 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-26T21:52:09,172 INFO [RS:1;828b3f090c91:42409 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-26T21:52:09,172 INFO [RS:0;828b3f090c91:33857 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-26T21:52:09,173 INFO [RS:0;828b3f090c91:33857 {}] hbase.ChoreService(168): Chore ScheduledChore name=828b3f090c91,33857,1732657928830-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-26T21:52:09,173 INFO [RS:1;828b3f090c91:42409 {}] hbase.ChoreService(168): Chore ScheduledChore name=828b3f090c91,42409,1732657928858-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-26T21:52:09,173 INFO [RS:1;828b3f090c91:42409 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-26T21:52:09,173 INFO [RS:0;828b3f090c91:33857 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-26T21:52:09,173 INFO [RS:1;828b3f090c91:42409 {}] regionserver.Replication(171): 828b3f090c91,42409,1732657928858 started 2024-11-26T21:52:09,173 INFO [RS:0;828b3f090c91:33857 {}] regionserver.Replication(171): 828b3f090c91,33857,1732657928830 started 2024-11-26T21:52:09,173 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7430f913-2694-54cf-261c-654b69c5fe06/de866367-466a-3788-a91b-60c9e5aa6ab2/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-26T21:52:09,174 INFO [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=59679324, jitterRate=-0.11070877313613892}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-26T21:52:09,174 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740: Writing region info on filesystem at 1732657929118Initializing all the Stores at 1732657929118Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732657929118Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732657929129 (+11 ms)Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732657929129Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732657929129Cleaning up temporary data from old regions at 1732657929165 (+36 ms)Region opened successfully at 1732657929174 (+9 ms) 2024-11-26T21:52:09,175 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-26T21:52:09,175 INFO [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-26T21:52:09,175 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-26T21:52:09,175 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-26T21:52:09,175 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-26T21:52:09,177 INFO [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-26T21:52:09,177 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1732657929174Disabling compacts and flushes for region at 1732657929174Disabling writes for close at 1732657929175 (+1 ms)Writing region close event to WAL at 1732657929176 (+1 ms)Closed at 1732657929176 2024-11-26T21:52:09,178 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-26T21:52:09,178 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta 2024-11-26T21:52:09,179 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-11-26T21:52:09,180 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-26T21:52:09,182 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2024-11-26T21:52:09,191 INFO [RS:0;828b3f090c91:33857 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-26T21:52:09,192 INFO [RS:0;828b3f090c91:33857 {}] regionserver.HRegionServer(1482): Serving as 828b3f090c91,33857,1732657928830, RpcServer on 828b3f090c91/172.17.0.2:33857, sessionid=0x1007e0536a90001 2024-11-26T21:52:09,192 DEBUG [RS:0;828b3f090c91:33857 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-26T21:52:09,192 DEBUG [RS:0;828b3f090c91:33857 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 828b3f090c91,33857,1732657928830 2024-11-26T21:52:09,192 DEBUG [RS:0;828b3f090c91:33857 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '828b3f090c91,33857,1732657928830' 2024-11-26T21:52:09,192 DEBUG [RS:0;828b3f090c91:33857 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-26T21:52:09,194 DEBUG [RS:0;828b3f090c91:33857 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-26T21:52:09,194 INFO [RS:1;828b3f090c91:42409 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-26T21:52:09,194 INFO [RS:1;828b3f090c91:42409 {}] regionserver.HRegionServer(1482): Serving as 828b3f090c91,42409,1732657928858, RpcServer on 828b3f090c91/172.17.0.2:42409, sessionid=0x1007e0536a90002 2024-11-26T21:52:09,194 DEBUG [RS:0;828b3f090c91:33857 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-26T21:52:09,194 DEBUG [RS:0;828b3f090c91:33857 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-26T21:52:09,194 DEBUG [RS:0;828b3f090c91:33857 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 828b3f090c91,33857,1732657928830 2024-11-26T21:52:09,194 DEBUG [RS:1;828b3f090c91:42409 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-26T21:52:09,194 DEBUG [RS:0;828b3f090c91:33857 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '828b3f090c91,33857,1732657928830' 2024-11-26T21:52:09,194 DEBUG [RS:0;828b3f090c91:33857 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-26T21:52:09,194 DEBUG [RS:1;828b3f090c91:42409 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 828b3f090c91,42409,1732657928858 2024-11-26T21:52:09,194 DEBUG [RS:1;828b3f090c91:42409 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '828b3f090c91,42409,1732657928858' 2024-11-26T21:52:09,194 DEBUG [RS:1;828b3f090c91:42409 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-26T21:52:09,195 DEBUG [RS:0;828b3f090c91:33857 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-26T21:52:09,195 DEBUG [RS:1;828b3f090c91:42409 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-26T21:52:09,195 DEBUG [RS:0;828b3f090c91:33857 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-26T21:52:09,195 INFO [RS:0;828b3f090c91:33857 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-26T21:52:09,195 INFO [RS:0;828b3f090c91:33857 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-26T21:52:09,195 DEBUG [RS:1;828b3f090c91:42409 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-26T21:52:09,195 DEBUG [RS:1;828b3f090c91:42409 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-26T21:52:09,196 DEBUG [RS:1;828b3f090c91:42409 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 828b3f090c91,42409,1732657928858 2024-11-26T21:52:09,196 DEBUG [RS:1;828b3f090c91:42409 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '828b3f090c91,42409,1732657928858' 2024-11-26T21:52:09,196 DEBUG [RS:1;828b3f090c91:42409 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-26T21:52:09,196 DEBUG [RS:1;828b3f090c91:42409 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-26T21:52:09,197 DEBUG [RS:1;828b3f090c91:42409 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-26T21:52:09,197 INFO [RS:1;828b3f090c91:42409 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-26T21:52:09,197 INFO [RS:1;828b3f090c91:42409 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-26T21:52:09,296 INFO [RS:0;828b3f090c91:33857 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-26T21:52:09,297 INFO [RS:1;828b3f090c91:42409 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-26T21:52:09,298 INFO [RS:0;828b3f090c91:33857 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=828b3f090c91%2C33857%2C1732657928830, suffix=, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7430f913-2694-54cf-261c-654b69c5fe06/de866367-466a-3788-a91b-60c9e5aa6ab2/WALs/828b3f090c91,33857,1732657928830, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7430f913-2694-54cf-261c-654b69c5fe06/de866367-466a-3788-a91b-60c9e5aa6ab2/oldWALs, maxLogs=55 2024-11-26T21:52:09,298 INFO [RS:1;828b3f090c91:42409 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=828b3f090c91%2C42409%2C1732657928858, suffix=, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7430f913-2694-54cf-261c-654b69c5fe06/de866367-466a-3788-a91b-60c9e5aa6ab2/WALs/828b3f090c91,42409,1732657928858, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7430f913-2694-54cf-261c-654b69c5fe06/de866367-466a-3788-a91b-60c9e5aa6ab2/oldWALs, maxLogs=55 2024-11-26T21:52:09,320 INFO [RS:1;828b3f090c91:42409 {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7430f913-2694-54cf-261c-654b69c5fe06/de866367-466a-3788-a91b-60c9e5aa6ab2/WALs/828b3f090c91,42409,1732657928858/828b3f090c91%2C42409%2C1732657928858.1732657929301 2024-11-26T21:52:09,321 DEBUG [RS:1;828b3f090c91:42409 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-11-26T21:52:09,326 INFO [RS:0;828b3f090c91:33857 {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7430f913-2694-54cf-261c-654b69c5fe06/de866367-466a-3788-a91b-60c9e5aa6ab2/WALs/828b3f090c91,33857,1732657928830/828b3f090c91%2C33857%2C1732657928830.1732657929301 2024-11-26T21:52:09,326 DEBUG [RS:0;828b3f090c91:33857 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-11-26T21:52:09,332 DEBUG [828b3f090c91:35155 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=2, allServersCount=2 2024-11-26T21:52:09,332 DEBUG [828b3f090c91:35155 {}] balancer.BalancerClusterState(204): Hosts are {828b3f090c91=0} racks are {/default-rack=0} 2024-11-26T21:52:09,333 DEBUG [828b3f090c91:35155 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-11-26T21:52:09,333 DEBUG [828b3f090c91:35155 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-11-26T21:52:09,333 DEBUG [828b3f090c91:35155 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-11-26T21:52:09,333 DEBUG [828b3f090c91:35155 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-11-26T21:52:09,333 INFO [828b3f090c91:35155 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-11-26T21:52:09,333 INFO [828b3f090c91:35155 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-11-26T21:52:09,333 DEBUG [828b3f090c91:35155 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-11-26T21:52:09,334 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=828b3f090c91,42409,1732657928858 2024-11-26T21:52:09,335 INFO [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 828b3f090c91,42409,1732657928858, state=OPENING 2024-11-26T21:52:09,336 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2024-11-26T21:52:09,337 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42409-0x1007e0536a90002, quorum=127.0.0.1:53598, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-26T21:52:09,337 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33857-0x1007e0536a90001, quorum=127.0.0.1:53598, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-26T21:52:09,337 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35155-0x1007e0536a90000, quorum=127.0.0.1:53598, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-26T21:52:09,338 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-26T21:52:09,338 DEBUG [PEWorker-3 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-26T21:52:09,338 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-26T21:52:09,338 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=828b3f090c91,42409,1732657928858}] 2024-11-26T21:52:09,339 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-26T21:52:09,491 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-26T21:52:09,495 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-8-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53669, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-26T21:52:09,499 INFO [RS_OPEN_META-regionserver/828b3f090c91:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-11-26T21:52:09,500 INFO [RS_OPEN_META-regionserver/828b3f090c91:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-26T21:52:09,500 INFO [RS_OPEN_META-regionserver/828b3f090c91:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-11-26T21:52:09,501 INFO [RS_OPEN_META-regionserver/828b3f090c91:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=828b3f090c91%2C42409%2C1732657928858.meta, suffix=.meta, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7430f913-2694-54cf-261c-654b69c5fe06/de866367-466a-3788-a91b-60c9e5aa6ab2/WALs/828b3f090c91,42409,1732657928858, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7430f913-2694-54cf-261c-654b69c5fe06/de866367-466a-3788-a91b-60c9e5aa6ab2/oldWALs, maxLogs=55 2024-11-26T21:52:09,523 INFO [RS_OPEN_META-regionserver/828b3f090c91:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7430f913-2694-54cf-261c-654b69c5fe06/de866367-466a-3788-a91b-60c9e5aa6ab2/WALs/828b3f090c91,42409,1732657928858/828b3f090c91%2C42409%2C1732657928858.meta.1732657929502.meta 2024-11-26T21:52:09,524 DEBUG [RS_OPEN_META-regionserver/828b3f090c91:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-11-26T21:52:09,524 DEBUG [RS_OPEN_META-regionserver/828b3f090c91:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-11-26T21:52:09,524 DEBUG [RS_OPEN_META-regionserver/828b3f090c91:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-11-26T21:52:09,524 DEBUG [RS_OPEN_META-regionserver/828b3f090c91:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-11-26T21:52:09,525 INFO [RS_OPEN_META-regionserver/828b3f090c91:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-11-26T21:52:09,525 DEBUG [RS_OPEN_META-regionserver/828b3f090c91:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-11-26T21:52:09,525 DEBUG [RS_OPEN_META-regionserver/828b3f090c91:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-26T21:52:09,525 DEBUG [RS_OPEN_META-regionserver/828b3f090c91:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-11-26T21:52:09,525 DEBUG [RS_OPEN_META-regionserver/828b3f090c91:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-11-26T21:52:09,526 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family info of region 1588230740 2024-11-26T21:52:09,527 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-26T21:52:09,527 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T21:52:09,527 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-26T21:52:09,528 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family ns of region 1588230740 2024-11-26T21:52:09,528 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-26T21:52:09,528 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T21:52:09,529 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-26T21:52:09,529 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family rep_barrier of region 1588230740 2024-11-26T21:52:09,529 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-26T21:52:09,529 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T21:52:09,530 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-26T21:52:09,530 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family table of region 1588230740 2024-11-26T21:52:09,530 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-26T21:52:09,530 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T21:52:09,530 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-26T21:52:09,531 DEBUG [RS_OPEN_META-regionserver/828b3f090c91:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-26T21:52:09,531 DEBUG [RS_OPEN_META-regionserver/828b3f090c91:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7430f913-2694-54cf-261c-654b69c5fe06/de866367-466a-3788-a91b-60c9e5aa6ab2/data/hbase/meta/1588230740 2024-11-26T21:52:09,532 DEBUG [RS_OPEN_META-regionserver/828b3f090c91:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7430f913-2694-54cf-261c-654b69c5fe06/de866367-466a-3788-a91b-60c9e5aa6ab2/data/hbase/meta/1588230740 2024-11-26T21:52:09,532 DEBUG [RS_OPEN_META-regionserver/828b3f090c91:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-26T21:52:09,532 DEBUG [RS_OPEN_META-regionserver/828b3f090c91:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-26T21:52:09,532 DEBUG [RS_OPEN_META-regionserver/828b3f090c91:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-26T21:52:09,533 DEBUG [RS_OPEN_META-regionserver/828b3f090c91:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-26T21:52:09,534 INFO [RS_OPEN_META-regionserver/828b3f090c91:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=75390938, jitterRate=0.12341251969337463}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-26T21:52:09,534 DEBUG [RS_OPEN_META-regionserver/828b3f090c91:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-11-26T21:52:09,534 DEBUG [RS_OPEN_META-regionserver/828b3f090c91:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1732657929525Writing region info on filesystem at 1732657929525Initializing all the Stores at 1732657929526 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732657929526Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732657929526Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732657929526Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732657929526Cleaning up temporary data from old regions at 1732657929532 (+6 ms)Running coprocessor post-open hooks at 1732657929534 (+2 ms)Region opened successfully at 1732657929534 2024-11-26T21:52:09,535 INFO [RS_OPEN_META-regionserver/828b3f090c91:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1732657929491 2024-11-26T21:52:09,538 DEBUG [RS_OPEN_META-regionserver/828b3f090c91:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-11-26T21:52:09,539 INFO [RS_OPEN_META-regionserver/828b3f090c91:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-11-26T21:52:09,539 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=828b3f090c91,42409,1732657928858 2024-11-26T21:52:09,540 INFO [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 828b3f090c91,42409,1732657928858, state=OPEN 2024-11-26T21:52:09,541 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35155-0x1007e0536a90000, quorum=127.0.0.1:53598, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-26T21:52:09,541 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42409-0x1007e0536a90002, quorum=127.0.0.1:53598, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-26T21:52:09,541 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33857-0x1007e0536a90001, quorum=127.0.0.1:53598, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-26T21:52:09,541 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-26T21:52:09,541 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=828b3f090c91,42409,1732657928858 2024-11-26T21:52:09,541 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-26T21:52:09,541 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-26T21:52:09,544 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2 2024-11-26T21:52:09,544 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=828b3f090c91,42409,1732657928858 in 203 msec 2024-11-26T21:52:09,546 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1 2024-11-26T21:52:09,546 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 366 msec 2024-11-26T21:52:09,546 DEBUG [PEWorker-2 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-26T21:52:09,546 INFO [PEWorker-2 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces 2024-11-26T21:52:09,550 DEBUG [PEWorker-2 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-26T21:52:09,550 DEBUG [PEWorker-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=828b3f090c91,42409,1732657928858, seqNum=-1] 2024-11-26T21:52:09,550 DEBUG [PEWorker-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-26T21:52:09,552 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-8-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41113, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-26T21:52:09,559 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 497 msec 2024-11-26T21:52:09,559 INFO [master/828b3f090c91:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1732657929559, completionTime=-1 2024-11-26T21:52:09,559 INFO [master/828b3f090c91:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=2; waited=0ms, expected min=2 server(s), max=2 server(s), master is running 2024-11-26T21:52:09,560 DEBUG [master/828b3f090c91:0:becomeActiveMaster {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-26T21:52:09,563 INFO [master/828b3f090c91:0:becomeActiveMaster {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-11-26T21:52:09,563 INFO [master/828b3f090c91:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732657989563 2024-11-26T21:52:09,563 INFO [master/828b3f090c91:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732658049563 2024-11-26T21:52:09,563 INFO [master/828b3f090c91:0:becomeActiveMaster {}] assignment.AssignmentManager(1783): Joined the cluster in 3 msec 2024-11-26T21:52:09,564 INFO [master/828b3f090c91:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=828b3f090c91,35155,1732657928779-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-26T21:52:09,564 INFO [master/828b3f090c91:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=828b3f090c91,35155,1732657928779-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-26T21:52:09,564 INFO [master/828b3f090c91:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=828b3f090c91,35155,1732657928779-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-26T21:52:09,564 INFO [master/828b3f090c91:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-828b3f090c91:35155, period=300000, unit=MILLISECONDS is enabled. 2024-11-26T21:52:09,564 INFO [master/828b3f090c91:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2024-11-26T21:52:09,565 INFO [master/828b3f090c91:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled. 2024-11-26T21:52:09,567 DEBUG [master/828b3f090c91:0.Chore.1 {}] janitor.CatalogJanitor(180): 2024-11-26T21:52:09,571 INFO [master/828b3f090c91:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 0.682sec 2024-11-26T21:52:09,571 INFO [master/828b3f090c91:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2024-11-26T21:52:09,572 INFO [master/828b3f090c91:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2024-11-26T21:52:09,572 INFO [master/828b3f090c91:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2024-11-26T21:52:09,572 INFO [master/828b3f090c91:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2024-11-26T21:52:09,572 INFO [master/828b3f090c91:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2024-11-26T21:52:09,572 INFO [master/828b3f090c91:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=828b3f090c91,35155,1732657928779-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-26T21:52:09,572 INFO [master/828b3f090c91:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=828b3f090c91,35155,1732657928779-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2024-11-26T21:52:09,573 DEBUG [master/828b3f090c91:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds 2024-11-26T21:52:09,573 INFO [master/828b3f090c91:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2024-11-26T21:52:09,573 INFO [master/828b3f090c91:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=828b3f090c91,35155,1732657928779-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-26T21:52:09,575 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6e62305b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T21:52:09,575 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 828b3f090c91,35155,-1 for getting cluster id 2024-11-26T21:52:09,575 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-26T21:52:09,577 DEBUG [HMaster-EventLoopGroup-6-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '45b5085c-0c14-4246-aa10-be0654098e00' 2024-11-26T21:52:09,578 DEBUG [RPCClient-NioEventLoopGroup-5-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-26T21:52:09,578 DEBUG [RPCClient-NioEventLoopGroup-5-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "45b5085c-0c14-4246-aa10-be0654098e00" 2024-11-26T21:52:09,579 DEBUG [RPCClient-NioEventLoopGroup-5-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@d678e66, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T21:52:09,579 DEBUG [RPCClient-NioEventLoopGroup-5-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [828b3f090c91,35155,-1] 2024-11-26T21:52:09,579 DEBUG [RPCClient-NioEventLoopGroup-5-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-26T21:52:09,579 DEBUG [RPCClient-NioEventLoopGroup-5-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T21:52:09,581 INFO [HMaster-EventLoopGroup-6-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44176, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-26T21:52:09,583 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@556fb9de, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T21:52:09,583 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-26T21:52:09,585 DEBUG [RPCClient-NioEventLoopGroup-5-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=828b3f090c91,42409,1732657928858, seqNum=-1] 2024-11-26T21:52:09,586 DEBUG [RPCClient-NioEventLoopGroup-5-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-26T21:52:09,587 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-8-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34744, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-26T21:52:09,590 INFO [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=828b3f090c91,35155,1732657928779 2024-11-26T21:52:09,590 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-26T21:52:09,592 DEBUG [RPCClient-NioEventLoopGroup-5-6 {}] client.AsyncConnectionImpl(321): The fetched master address is 828b3f090c91,35155,1732657928779 2024-11-26T21:52:09,592 DEBUG [RPCClient-NioEventLoopGroup-5-6 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@6f72830e 2024-11-26T21:52:09,592 DEBUG [RPCClient-NioEventLoopGroup-5-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-26T21:52:09,594 INFO [HMaster-EventLoopGroup-6-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44190, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-26T21:52:09,595 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35155 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'testBlockEvictionOnGracefulStop1732657929590', {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-26T21:52:09,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35155 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=testBlockEvictionOnGracefulStop1732657929590 2024-11-26T21:52:09,598 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=testBlockEvictionOnGracefulStop1732657929590 execute state=CREATE_TABLE_PRE_OPERATION 2024-11-26T21:52:09,598 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T21:52:09,598 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35155 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "testBlockEvictionOnGracefulStop1732657929590" procId is: 4 2024-11-26T21:52:09,599 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=testBlockEvictionOnGracefulStop1732657929590 execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-26T21:52:09,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35155 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-26T21:52:09,608 INFO [RegionOpenAndInit-testBlockEvictionOnGracefulStop1732657929590-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 282fdb3a3c29b93634ce02b236635dc2, NAME => 'testBlockEvictionOnGracefulStop1732657929590,,1732657929595.282fdb3a3c29b93634ce02b236635dc2.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testBlockEvictionOnGracefulStop1732657929590', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7430f913-2694-54cf-261c-654b69c5fe06/de866367-466a-3788-a91b-60c9e5aa6ab2 2024-11-26T21:52:09,614 DEBUG [RegionOpenAndInit-testBlockEvictionOnGracefulStop1732657929590-pool-0 {}] regionserver.HRegion(898): Instantiated testBlockEvictionOnGracefulStop1732657929590,,1732657929595.282fdb3a3c29b93634ce02b236635dc2.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-26T21:52:09,614 DEBUG [RegionOpenAndInit-testBlockEvictionOnGracefulStop1732657929590-pool-0 {}] regionserver.HRegion(1722): Closing 282fdb3a3c29b93634ce02b236635dc2, disabling compactions & flushes 2024-11-26T21:52:09,614 INFO [RegionOpenAndInit-testBlockEvictionOnGracefulStop1732657929590-pool-0 {}] regionserver.HRegion(1755): Closing region testBlockEvictionOnGracefulStop1732657929590,,1732657929595.282fdb3a3c29b93634ce02b236635dc2. 2024-11-26T21:52:09,614 DEBUG [RegionOpenAndInit-testBlockEvictionOnGracefulStop1732657929590-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on testBlockEvictionOnGracefulStop1732657929590,,1732657929595.282fdb3a3c29b93634ce02b236635dc2. 2024-11-26T21:52:09,614 DEBUG [RegionOpenAndInit-testBlockEvictionOnGracefulStop1732657929590-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on testBlockEvictionOnGracefulStop1732657929590,,1732657929595.282fdb3a3c29b93634ce02b236635dc2. after waiting 0 ms 2024-11-26T21:52:09,614 DEBUG [RegionOpenAndInit-testBlockEvictionOnGracefulStop1732657929590-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region testBlockEvictionOnGracefulStop1732657929590,,1732657929595.282fdb3a3c29b93634ce02b236635dc2. 2024-11-26T21:52:09,614 INFO [RegionOpenAndInit-testBlockEvictionOnGracefulStop1732657929590-pool-0 {}] regionserver.HRegion(1973): Closed testBlockEvictionOnGracefulStop1732657929590,,1732657929595.282fdb3a3c29b93634ce02b236635dc2. 2024-11-26T21:52:09,614 DEBUG [RegionOpenAndInit-testBlockEvictionOnGracefulStop1732657929590-pool-0 {}] regionserver.HRegion(1676): Region close journal for 282fdb3a3c29b93634ce02b236635dc2: Waiting for close lock at 1732657929614Disabling compacts and flushes for region at 1732657929614Disabling writes for close at 1732657929614Writing region close event to WAL at 1732657929614Closed at 1732657929614 2024-11-26T21:52:09,616 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=testBlockEvictionOnGracefulStop1732657929590 execute state=CREATE_TABLE_ADD_TO_META 2024-11-26T21:52:09,616 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testBlockEvictionOnGracefulStop1732657929590,,1732657929595.282fdb3a3c29b93634ce02b236635dc2.","families":{"info":[{"qualifier":"regioninfo","vlen":78,"tag":[],"timestamp":"1732657929616"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732657929616"}]},"ts":"1732657929616"} 2024-11-26T21:52:09,620 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-26T21:52:09,621 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=testBlockEvictionOnGracefulStop1732657929590 execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-26T21:52:09,621 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testBlockEvictionOnGracefulStop1732657929590","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732657929621"}]},"ts":"1732657929621"} 2024-11-26T21:52:09,624 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=testBlockEvictionOnGracefulStop1732657929590, state=ENABLING in hbase:meta 2024-11-26T21:52:09,625 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(204): Hosts are {828b3f090c91=0} racks are {/default-rack=0} 2024-11-26T21:52:09,625 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-11-26T21:52:09,625 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-11-26T21:52:09,625 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-11-26T21:52:09,625 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-11-26T21:52:09,625 INFO [PEWorker-3 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-11-26T21:52:09,625 INFO [PEWorker-3 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-11-26T21:52:09,625 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-11-26T21:52:09,626 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1732657929590, region=282fdb3a3c29b93634ce02b236635dc2, ASSIGN}] 2024-11-26T21:52:09,627 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1732657929590, region=282fdb3a3c29b93634ce02b236635dc2, ASSIGN 2024-11-26T21:52:09,628 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1732657929590, region=282fdb3a3c29b93634ce02b236635dc2, ASSIGN; state=OFFLINE, location=828b3f090c91,33857,1732657928830; forceNewPlan=false, retain=false 2024-11-26T21:52:09,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35155 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-26T21:52:09,778 INFO [828b3f090c91:35155 {}] balancer.BaseLoadBalancer(388): Reassigned 1 regions. 1 retained the pre-restart assignment. 2024-11-26T21:52:09,779 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=282fdb3a3c29b93634ce02b236635dc2, regionState=OPENING, regionLocation=828b3f090c91,33857,1732657928830 2024-11-26T21:52:09,783 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-8-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1732657929590, region=282fdb3a3c29b93634ce02b236635dc2, ASSIGN because future has completed 2024-11-26T21:52:09,784 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 282fdb3a3c29b93634ce02b236635dc2, server=828b3f090c91,33857,1732657928830}] 2024-11-26T21:52:09,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35155 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-26T21:52:09,938 DEBUG [RSProcedureDispatcher-pool-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-26T21:52:09,943 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-7-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35659, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-26T21:52:09,947 INFO [RS_OPEN_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(132): Open testBlockEvictionOnGracefulStop1732657929590,,1732657929595.282fdb3a3c29b93634ce02b236635dc2. 2024-11-26T21:52:09,948 DEBUG [RS_OPEN_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7752): Opening region: {ENCODED => 282fdb3a3c29b93634ce02b236635dc2, NAME => 'testBlockEvictionOnGracefulStop1732657929590,,1732657929595.282fdb3a3c29b93634ce02b236635dc2.', STARTKEY => '', ENDKEY => ''} 2024-11-26T21:52:09,948 DEBUG [RS_OPEN_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testBlockEvictionOnGracefulStop1732657929590 282fdb3a3c29b93634ce02b236635dc2 2024-11-26T21:52:09,949 DEBUG [RS_OPEN_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(898): Instantiated testBlockEvictionOnGracefulStop1732657929590,,1732657929595.282fdb3a3c29b93634ce02b236635dc2.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-26T21:52:09,949 DEBUG [RS_OPEN_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7794): checking encryption for 282fdb3a3c29b93634ce02b236635dc2 2024-11-26T21:52:09,949 DEBUG [RS_OPEN_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7797): checking classloading for 282fdb3a3c29b93634ce02b236635dc2 2024-11-26T21:52:09,950 INFO [StoreOpener-282fdb3a3c29b93634ce02b236635dc2-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family family of region 282fdb3a3c29b93634ce02b236635dc2 2024-11-26T21:52:09,955 INFO [StoreOpener-282fdb3a3c29b93634ce02b236635dc2-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 282fdb3a3c29b93634ce02b236635dc2 columnFamilyName family 2024-11-26T21:52:09,955 DEBUG [StoreOpener-282fdb3a3c29b93634ce02b236635dc2-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T21:52:09,955 INFO [StoreOpener-282fdb3a3c29b93634ce02b236635dc2-1 {}] regionserver.HStore(327): Store=282fdb3a3c29b93634ce02b236635dc2/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-26T21:52:09,956 DEBUG [RS_OPEN_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1038): replaying wal for 282fdb3a3c29b93634ce02b236635dc2 2024-11-26T21:52:09,957 DEBUG [RS_OPEN_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7430f913-2694-54cf-261c-654b69c5fe06/de866367-466a-3788-a91b-60c9e5aa6ab2/data/default/testBlockEvictionOnGracefulStop1732657929590/282fdb3a3c29b93634ce02b236635dc2 2024-11-26T21:52:09,957 DEBUG [RS_OPEN_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7430f913-2694-54cf-261c-654b69c5fe06/de866367-466a-3788-a91b-60c9e5aa6ab2/data/default/testBlockEvictionOnGracefulStop1732657929590/282fdb3a3c29b93634ce02b236635dc2 2024-11-26T21:52:09,957 DEBUG [RS_OPEN_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1048): stopping wal replay for 282fdb3a3c29b93634ce02b236635dc2 2024-11-26T21:52:09,957 DEBUG [RS_OPEN_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1060): Cleaning up temporary data for 282fdb3a3c29b93634ce02b236635dc2 2024-11-26T21:52:09,958 DEBUG [RS_OPEN_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1093): writing seq id for 282fdb3a3c29b93634ce02b236635dc2 2024-11-26T21:52:09,965 DEBUG [RS_OPEN_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7430f913-2694-54cf-261c-654b69c5fe06/de866367-466a-3788-a91b-60c9e5aa6ab2/data/default/testBlockEvictionOnGracefulStop1732657929590/282fdb3a3c29b93634ce02b236635dc2/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-26T21:52:09,966 INFO [RS_OPEN_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1114): Opened 282fdb3a3c29b93634ce02b236635dc2; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=72660645, jitterRate=0.08272798359394073}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-26T21:52:09,966 DEBUG [RS_OPEN_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 282fdb3a3c29b93634ce02b236635dc2 2024-11-26T21:52:09,967 DEBUG [RS_OPEN_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1006): Region open journal for 282fdb3a3c29b93634ce02b236635dc2: Running coprocessor pre-open hook at 1732657929949Writing region info on filesystem at 1732657929949Initializing all the Stores at 1732657929950 (+1 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732657929950Cleaning up temporary data from old regions at 1732657929957 (+7 ms)Running coprocessor post-open hooks at 1732657929966 (+9 ms)Region opened successfully at 1732657929967 (+1 ms) 2024-11-26T21:52:09,968 INFO [RS_OPEN_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2236): Post open deploy tasks for testBlockEvictionOnGracefulStop1732657929590,,1732657929595.282fdb3a3c29b93634ce02b236635dc2., pid=6, masterSystemTime=1732657929937 2024-11-26T21:52:09,971 DEBUG [RS_OPEN_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2266): Finished post open deploy task for testBlockEvictionOnGracefulStop1732657929590,,1732657929595.282fdb3a3c29b93634ce02b236635dc2. 2024-11-26T21:52:09,972 INFO [RS_OPEN_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(153): Opened testBlockEvictionOnGracefulStop1732657929590,,1732657929595.282fdb3a3c29b93634ce02b236635dc2. 2024-11-26T21:52:09,972 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=282fdb3a3c29b93634ce02b236635dc2, regionState=OPEN, openSeqNum=2, regionLocation=828b3f090c91,33857,1732657928830 2024-11-26T21:52:09,975 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-8-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 282fdb3a3c29b93634ce02b236635dc2, server=828b3f090c91,33857,1732657928830 because future has completed 2024-11-26T21:52:09,978 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=6, resume processing ppid=5 2024-11-26T21:52:09,978 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure 282fdb3a3c29b93634ce02b236635dc2, server=828b3f090c91,33857,1732657928830 in 192 msec 2024-11-26T21:52:09,980 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=5, resume processing ppid=4 2024-11-26T21:52:09,980 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1732657929590, region=282fdb3a3c29b93634ce02b236635dc2, ASSIGN in 352 msec 2024-11-26T21:52:09,980 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=testBlockEvictionOnGracefulStop1732657929590 execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-26T21:52:09,981 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testBlockEvictionOnGracefulStop1732657929590","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732657929980"}]},"ts":"1732657929980"} 2024-11-26T21:52:09,983 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=testBlockEvictionOnGracefulStop1732657929590, state=ENABLED in hbase:meta 2024-11-26T21:52:09,983 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=testBlockEvictionOnGracefulStop1732657929590 execute state=CREATE_TABLE_POST_OPERATION 2024-11-26T21:52:09,985 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=testBlockEvictionOnGracefulStop1732657929590 in 388 msec 2024-11-26T21:52:10,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35155 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-26T21:52:10,229 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table testBlockEvictionOnGracefulStop1732657929590 get assigned. Timeout = 60000ms 2024-11-26T21:52:10,228 INFO [RPCClient-NioEventLoopGroup-5-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:testBlockEvictionOnGracefulStop1732657929590 completed 2024-11-26T21:52:10,229 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-26T21:52:10,237 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table testBlockEvictionOnGracefulStop1732657929590 assigned to meta. Checking AM states. 2024-11-26T21:52:10,237 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-26T21:52:10,237 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table testBlockEvictionOnGracefulStop1732657929590 assigned. 2024-11-26T21:52:10,241 DEBUG [RPCClient-NioEventLoopGroup-5-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testBlockEvictionOnGracefulStop1732657929590', row='row1', locateType=CURRENT is [region=testBlockEvictionOnGracefulStop1732657929590,,1732657929595.282fdb3a3c29b93634ce02b236635dc2., hostname=828b3f090c91,33857,1732657928830, seqNum=2] 2024-11-26T21:52:10,241 DEBUG [RPCClient-NioEventLoopGroup-5-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-26T21:52:10,243 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-7-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50890, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-26T21:52:10,248 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 282fdb3a3c29b93634ce02b236635dc2 1/1 column families, dataSize=78 B heapSize=480 B 2024-11-26T21:52:10,257 DEBUG [Time-limited test {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 2ae1dddb00634a5681fd7785f8291bc0 2024-11-26T21:52:10,271 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7430f913-2694-54cf-261c-654b69c5fe06/de866367-466a-3788-a91b-60c9e5aa6ab2/data/default/testBlockEvictionOnGracefulStop1732657929590/282fdb3a3c29b93634ce02b236635dc2/.tmp/family/2ae1dddb00634a5681fd7785f8291bc0 is 43, key is row1/family:qf1/1/Put/seqid=0 2024-11-26T21:52:10,272 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=78 B at sequenceid=6 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7430f913-2694-54cf-261c-654b69c5fe06/de866367-466a-3788-a91b-60c9e5aa6ab2/data/default/testBlockEvictionOnGracefulStop1732657929590/282fdb3a3c29b93634ce02b236635dc2/.tmp/family/2ae1dddb00634a5681fd7785f8291bc0 2024-11-26T21:52:10,275 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7430f913-2694-54cf-261c-654b69c5fe06/de866367-466a-3788-a91b-60c9e5aa6ab2/data/default/testBlockEvictionOnGracefulStop1732657929590/282fdb3a3c29b93634ce02b236635dc2/.tmp/family/2ae1dddb00634a5681fd7785f8291bc0 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7430f913-2694-54cf-261c-654b69c5fe06/de866367-466a-3788-a91b-60c9e5aa6ab2/data/default/testBlockEvictionOnGracefulStop1732657929590/282fdb3a3c29b93634ce02b236635dc2/family/2ae1dddb00634a5681fd7785f8291bc0 2024-11-26T21:52:10,276 DEBUG [Time-limited test {}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7430f913-2694-54cf-261c-654b69c5fe06/de866367-466a-3788-a91b-60c9e5aa6ab2/data/default/testBlockEvictionOnGracefulStop1732657929590/282fdb3a3c29b93634ce02b236635dc2/family/2ae1dddb00634a5681fd7785f8291bc0, delay=901 ms 2024-11-26T21:52:10,277 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7430f913-2694-54cf-261c-654b69c5fe06/de866367-466a-3788-a91b-60c9e5aa6ab2/data/default/testBlockEvictionOnGracefulStop1732657929590/282fdb3a3c29b93634ce02b236635dc2/family/2ae1dddb00634a5681fd7785f8291bc0, entries=2, sequenceid=6, filesize=4.9 K 2024-11-26T21:52:10,277 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~78 B/78, heapSize ~464 B/464, currentSize=0 B/0 for 282fdb3a3c29b93634ce02b236635dc2 in 29ms, sequenceid=6, compaction requested=false 2024-11-26T21:52:10,278 DEBUG [Time-limited test {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'testBlockEvictionOnGracefulStop1732657929590' 2024-11-26T21:52:10,278 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for 282fdb3a3c29b93634ce02b236635dc2: 2024-11-26T21:52:11,043 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-26T21:52:11,178 DEBUG [hfile-prefetch-1732657928243 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 2ae1dddb00634a5681fd7785f8291bc0_0. 2024-11-26T21:52:11,180 DEBUG [hfile-prefetch-1732657928243 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 2ae1dddb00634a5681fd7785f8291bc0_117. 2024-11-26T21:52:11,181 DEBUG [hfile-prefetch-1732657928243 {}] bucket.BucketCache(2281): Notifying caching completed for file file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7430f913-2694-54cf-261c-654b69c5fe06/de866367-466a-3788-a91b-60c9e5aa6ab2/data/default/testBlockEvictionOnGracefulStop1732657929590/282fdb3a3c29b93634ce02b236635dc2/family/2ae1dddb00634a5681fd7785f8291bc0, with total blocks 2, and data blocks 1 2024-11-26T21:52:11,181 DEBUG [hfile-prefetch-1732657928243 {}] bucket.BucketCache(2285): iterating over 0 entries in the backing map 2024-11-26T21:52:11,182 DEBUG [hfile-prefetch-1732657928243 {}] bucket.BucketCache(2308): Prefetch executor completed for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7430f913-2694-54cf-261c-654b69c5fe06/de866367-466a-3788-a91b-60c9e5aa6ab2/data/default/testBlockEvictionOnGracefulStop1732657929590/282fdb3a3c29b93634ce02b236635dc2/family/2ae1dddb00634a5681fd7785f8291bc0, but only 0 data blocks were cached. Total data blocks for file: 1. Checking for blocks pending cache in cache writer queue. 2024-11-26T21:52:11,182 INFO [hfile-prefetch-1732657928243 {}] bucket.BucketCache(2331): We found only 0 data blocks cached from a total of 1 for file file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7430f913-2694-54cf-261c-654b69c5fe06/de866367-466a-3788-a91b-60c9e5aa6ab2/data/default/testBlockEvictionOnGracefulStop1732657929590/282fdb3a3c29b93634ce02b236635dc2/family/2ae1dddb00634a5681fd7785f8291bc0, but no blocks pending caching. Maybe cache is full or evictions happened concurrently to cache prefetch. 2024-11-26T21:52:11,182 DEBUG [hfile-prefetch-1732657928243 {}] hfile.PrefetchExecutor(121): Prefetch completed for 2ae1dddb00634a5681fd7785f8291bc0 2024-11-26T21:52:11,240 DEBUG [bucket-cache-persister {}] bucket.BucketCachePersister(47): Cache is inconsistent, persisting to disk 2024-11-26T21:52:11,240 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1487): Thread bucket-cache-persister started persisting bucket cache to file 2024-11-26T21:52:11,240 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1494): Persist in new chunked persistence format. 2024-11-26T21:52:11,240 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1673): persistToFile: before persisting backing map size: 1, fullycachedFiles size: 0, chunkSize: 10000 2024-11-26T21:52:11,247 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1680): persistToFile: after persisting backing map size: 1, fullycachedFiles size: 0 2024-11-26T21:52:11,247 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1498): PersistToFile: after persisting backing map size: 1, fullycachedFiles size: 0, file name: bucket.persistence1732657931240 2024-11-26T21:52:11,247 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1509): Thread bucket-cache-persister finished persisting bucket cache to file, renaming 2024-11-26T21:52:11,278 INFO [Time-limited test {}] hbase.SingleProcessHBaseCluster(504): Stopping Thread[RS:0;828b3f090c91:33857,5,FailOnTimeoutGroup] 2024-11-26T21:52:11,279 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '828b3f090c91,33857,1732657928830' ***** 2024-11-26T21:52:11,279 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Stopping rs 0 2024-11-26T21:52:11,279 INFO [RS:0;828b3f090c91:33857 {}] bucket.BucketCache(1738): Disabling cache 2024-11-26T21:52:11,279 INFO [RS:0;828b3f090c91:33857 {}] bucket.BucketCache(1762): Shutdown bucket cache: IO persistent=true; path to write=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/b34d8522-cb7c-4d9b-6a0e-fe88ad6cba6d/bucket.persistence 2024-11-26T21:52:11,279 INFO [Time-limited test-BucketCacheWriter-0 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-0 exiting, cacheEnabled=false 2024-11-26T21:52:11,279 INFO [Time-limited test-BucketCacheWriter-1 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-1 exiting, cacheEnabled=false 2024-11-26T21:52:11,279 INFO [Time-limited test-BucketCacheWriter-2 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-2 exiting, cacheEnabled=false 2024-11-26T21:52:11,279 INFO [RS:0;828b3f090c91:33857 {}] bucket.BucketCache(1768): Shutting down cache persister thread. 2024-11-26T21:52:11,279 WARN [bucket-cache-persister {}] bucket.BucketCachePersister(64): Interrupting BucketCachePersister thread. java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCachePersister.run(BucketCachePersister.java:45) ~[classes/:?] 2024-11-26T21:52:11,289 DEBUG [RS:0;828b3f090c91:33857 {}] bucket.BucketCache(1487): Thread RS:0;828b3f090c91:33857 started persisting bucket cache to file 2024-11-26T21:52:11,290 DEBUG [RS:0;828b3f090c91:33857 {}] bucket.BucketCache(1494): Persist in new chunked persistence format. 2024-11-26T21:52:11,290 DEBUG [RS:0;828b3f090c91:33857 {}] bucket.BucketCache(1673): persistToFile: before persisting backing map size: 1, fullycachedFiles size: 0, chunkSize: 10000 2024-11-26T21:52:11,292 DEBUG [RS:0;828b3f090c91:33857 {}] bucket.BucketCache(1680): persistToFile: after persisting backing map size: 1, fullycachedFiles size: 0 2024-11-26T21:52:11,293 DEBUG [RS:0;828b3f090c91:33857 {}] bucket.BucketCache(1498): PersistToFile: after persisting backing map size: 1, fullycachedFiles size: 0, file name: bucket.persistence1732657931289 2024-11-26T21:52:11,293 DEBUG [RS:0;828b3f090c91:33857 {}] bucket.BucketCache(1509): Thread RS:0;828b3f090c91:33857 finished persisting bucket cache to file, renaming 2024-11-26T21:52:11,293 INFO [RS:0;828b3f090c91:33857 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-26T21:52:11,293 INFO [RS:0;828b3f090c91:33857 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-26T21:52:11,293 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-26T21:52:11,293 INFO [RS:0;828b3f090c91:33857 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-26T21:52:11,293 INFO [RS:0;828b3f090c91:33857 {}] regionserver.HRegionServer(3091): Received CLOSE for 282fdb3a3c29b93634ce02b236635dc2 2024-11-26T21:52:11,293 INFO [RS:0;828b3f090c91:33857 {}] regionserver.HRegionServer(959): stopping server 828b3f090c91,33857,1732657928830 2024-11-26T21:52:11,293 INFO [RS:0;828b3f090c91:33857 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-26T21:52:11,293 INFO [RS:0;828b3f090c91:33857 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;828b3f090c91:33857. 2024-11-26T21:52:11,294 INFO [RS_CLOSE_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1662): Closing region testBlockEvictionOnGracefulStop1732657929590,,1732657929595.282fdb3a3c29b93634ce02b236635dc2. during a graceful stop, and cache persistence is on, so setting evict on close to false. 2024-11-26T21:52:11,294 DEBUG [RS_CLOSE_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 282fdb3a3c29b93634ce02b236635dc2, disabling compactions & flushes 2024-11-26T21:52:11,294 DEBUG [RS:0;828b3f090c91:33857 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-26T21:52:11,294 DEBUG [RS:0;828b3f090c91:33857 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T21:52:11,294 INFO [RS_CLOSE_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testBlockEvictionOnGracefulStop1732657929590,,1732657929595.282fdb3a3c29b93634ce02b236635dc2. 2024-11-26T21:52:11,294 DEBUG [RS_CLOSE_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testBlockEvictionOnGracefulStop1732657929590,,1732657929595.282fdb3a3c29b93634ce02b236635dc2. 2024-11-26T21:52:11,294 DEBUG [RS_CLOSE_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testBlockEvictionOnGracefulStop1732657929590,,1732657929595.282fdb3a3c29b93634ce02b236635dc2. after waiting 0 ms 2024-11-26T21:52:11,294 INFO [RS:0;828b3f090c91:33857 {}] regionserver.HRegionServer(1321): Waiting on 1 regions to close 2024-11-26T21:52:11,294 DEBUG [RS_CLOSE_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testBlockEvictionOnGracefulStop1732657929590,,1732657929595.282fdb3a3c29b93634ce02b236635dc2. 2024-11-26T21:52:11,294 DEBUG [RS:0;828b3f090c91:33857 {}] regionserver.HRegionServer(1325): Online Regions={282fdb3a3c29b93634ce02b236635dc2=testBlockEvictionOnGracefulStop1732657929590,,1732657929595.282fdb3a3c29b93634ce02b236635dc2.} 2024-11-26T21:52:11,294 DEBUG [RS:0;828b3f090c91:33857 {}] regionserver.HRegionServer(1351): Waiting on 282fdb3a3c29b93634ce02b236635dc2 2024-11-26T21:52:11,298 DEBUG [RS_CLOSE_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7430f913-2694-54cf-261c-654b69c5fe06/de866367-466a-3788-a91b-60c9e5aa6ab2/data/default/testBlockEvictionOnGracefulStop1732657929590/282fdb3a3c29b93634ce02b236635dc2/recovered.edits/9.seqid, newMaxSeqId=9, maxSeqId=1 2024-11-26T21:52:11,298 INFO [RS_CLOSE_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testBlockEvictionOnGracefulStop1732657929590,,1732657929595.282fdb3a3c29b93634ce02b236635dc2. 2024-11-26T21:52:11,298 DEBUG [RS_CLOSE_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 282fdb3a3c29b93634ce02b236635dc2: Waiting for close lock at 1732657931293Running coprocessor pre-close hooks at 1732657931294 (+1 ms)Disabling compacts and flushes for region at 1732657931294Disabling writes for close at 1732657931294Writing region close event to WAL at 1732657931295 (+1 ms)Running coprocessor post-close hooks at 1732657931298 (+3 ms)Closed at 1732657931298 2024-11-26T21:52:11,299 DEBUG [RS_CLOSE_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testBlockEvictionOnGracefulStop1732657929590,,1732657929595.282fdb3a3c29b93634ce02b236635dc2. 2024-11-26T21:52:11,356 INFO [regionserver/828b3f090c91:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-26T21:52:11,494 INFO [RS:0;828b3f090c91:33857 {}] regionserver.HRegionServer(976): stopping server 828b3f090c91,33857,1732657928830; all regions closed. 2024-11-26T21:52:11,500 DEBUG [RS:0;828b3f090c91:33857 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7430f913-2694-54cf-261c-654b69c5fe06/de866367-466a-3788-a91b-60c9e5aa6ab2/oldWALs 2024-11-26T21:52:11,500 INFO [RS:0;828b3f090c91:33857 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 828b3f090c91%2C33857%2C1732657928830:(num 1732657929301) 2024-11-26T21:52:11,500 DEBUG [RS:0;828b3f090c91:33857 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T21:52:11,500 INFO [RS:0;828b3f090c91:33857 {}] regionserver.LeaseManager(133): Closed leases 2024-11-26T21:52:11,501 INFO [RS:0;828b3f090c91:33857 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-26T21:52:11,501 INFO [RS:0;828b3f090c91:33857 {}] hbase.ChoreService(370): Chore service for: regionserver/828b3f090c91:0 had [ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-11-26T21:52:11,502 INFO [RS:0;828b3f090c91:33857 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-26T21:52:11,502 INFO [RS:0;828b3f090c91:33857 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-26T21:52:11,502 INFO [regionserver/828b3f090c91:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-26T21:52:11,502 INFO [RS:0;828b3f090c91:33857 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-26T21:52:11,502 INFO [RS:0;828b3f090c91:33857 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-26T21:52:11,503 INFO [RS:0;828b3f090c91:33857 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:33857 2024-11-26T21:52:11,507 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33857-0x1007e0536a90001, quorum=127.0.0.1:53598, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/828b3f090c91,33857,1732657928830 2024-11-26T21:52:11,507 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35155-0x1007e0536a90000, quorum=127.0.0.1:53598, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-26T21:52:11,507 INFO [RS:0;828b3f090c91:33857 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-26T21:52:11,508 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [828b3f090c91,33857,1732657928830] 2024-11-26T21:52:11,509 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/828b3f090c91,33857,1732657928830 already deleted, retry=false 2024-11-26T21:52:11,509 INFO [RegionServerTracker-0 {}] master.ServerManager(695): Processing expiration of 828b3f090c91,33857,1732657928830 on 828b3f090c91,35155,1732657928779 2024-11-26T21:52:11,514 DEBUG [RegionServerTracker-0 {}] procedure2.ProcedureExecutor(1139): Stored pid=7, state=RUNNABLE:SERVER_CRASH_START, hasLock=false; ServerCrashProcedure 828b3f090c91,33857,1732657928830, splitWal=true, meta=false 2024-11-26T21:52:11,516 INFO [RegionServerTracker-0 {}] assignment.AssignmentManager(1999): Scheduled ServerCrashProcedure pid=7 for 828b3f090c91,33857,1732657928830 (carryingMeta=false) 828b3f090c91,33857,1732657928830/CRASHED/regionCount=1/lock=java.util.concurrent.locks.ReentrantReadWriteLock@ec43f25[Write locks = 1, Read locks = 0], oldState=ONLINE. 2024-11-26T21:52:11,516 INFO [PEWorker-1 {}] procedure.ServerCrashProcedure(169): Start pid=7, state=RUNNABLE:SERVER_CRASH_START, hasLock=true; ServerCrashProcedure 828b3f090c91,33857,1732657928830, splitWal=true, meta=false 2024-11-26T21:52:11,517 INFO [PEWorker-1 {}] procedure.ServerCrashProcedure(207): 828b3f090c91,33857,1732657928830 had 1 regions 2024-11-26T21:52:11,518 INFO [PEWorker-1 {}] procedure.ServerCrashProcedure(339): Splitting WALs pid=7, state=RUNNABLE:SERVER_CRASH_SPLIT_LOGS, hasLock=true; ServerCrashProcedure 828b3f090c91,33857,1732657928830, splitWal=true, meta=false, isMeta: false 2024-11-26T21:52:11,519 DEBUG [PEWorker-1 {}] master.MasterWalManager(329): Renamed region directory: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7430f913-2694-54cf-261c-654b69c5fe06/de866367-466a-3788-a91b-60c9e5aa6ab2/WALs/828b3f090c91,33857,1732657928830-splitting 2024-11-26T21:52:11,519 INFO [PEWorker-1 {}] master.SplitLogManager(171): file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7430f913-2694-54cf-261c-654b69c5fe06/de866367-466a-3788-a91b-60c9e5aa6ab2/WALs/828b3f090c91,33857,1732657928830-splitting dir is empty, no logs to split. 2024-11-26T21:52:11,519 INFO [PEWorker-1 {}] master.SplitWALManager(105): 828b3f090c91,33857,1732657928830 WAL count=0, meta=false 2024-11-26T21:52:11,521 INFO [PEWorker-1 {}] master.SplitLogManager(171): file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7430f913-2694-54cf-261c-654b69c5fe06/de866367-466a-3788-a91b-60c9e5aa6ab2/WALs/828b3f090c91,33857,1732657928830-splitting dir is empty, no logs to split. 2024-11-26T21:52:11,521 INFO [PEWorker-1 {}] master.SplitWALManager(105): 828b3f090c91,33857,1732657928830 WAL count=0, meta=false 2024-11-26T21:52:11,521 DEBUG [PEWorker-1 {}] procedure.ServerCrashProcedure(329): Check if 828b3f090c91,33857,1732657928830 WAL splitting is done? wals=0, meta=false 2024-11-26T21:52:11,522 INFO [PEWorker-1 {}] procedure.ServerCrashProcedure(321): Remove WAL directory for 828b3f090c91,33857,1732657928830 failed, ignore...File file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7430f913-2694-54cf-261c-654b69c5fe06/de866367-466a-3788-a91b-60c9e5aa6ab2/WALs/828b3f090c91,33857,1732657928830-splitting does not exist 2024-11-26T21:52:11,524 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=8, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1732657929590, region=282fdb3a3c29b93634ce02b236635dc2, ASSIGN}] 2024-11-26T21:52:11,525 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=8, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1732657929590, region=282fdb3a3c29b93634ce02b236635dc2, ASSIGN 2024-11-26T21:52:11,525 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(269): Starting pid=8, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1732657929590, region=282fdb3a3c29b93634ce02b236635dc2, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-11-26T21:52:11,609 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33857-0x1007e0536a90001, quorum=127.0.0.1:53598, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-26T21:52:11,609 INFO [RS:0;828b3f090c91:33857 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-26T21:52:11,609 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33857-0x1007e0536a90001, quorum=127.0.0.1:53598, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-26T21:52:11,609 INFO [RS:0;828b3f090c91:33857 {}] regionserver.HRegionServer(1031): Exiting; stopping=828b3f090c91,33857,1732657928830; zookeeper connection closed. 2024-11-26T21:52:11,610 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@2b71cac {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@2b71cac 2024-11-26T21:52:11,677 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=8 updating hbase:meta row=282fdb3a3c29b93634ce02b236635dc2, regionState=OPENING, regionLocation=828b3f090c91,42409,1732657928858 2024-11-26T21:52:11,682 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-8-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=8, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1732657929590, region=282fdb3a3c29b93634ce02b236635dc2, ASSIGN because future has completed 2024-11-26T21:52:11,684 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=9, ppid=8, state=RUNNABLE, hasLock=false; OpenRegionProcedure 282fdb3a3c29b93634ce02b236635dc2, server=828b3f090c91,42409,1732657928858}] 2024-11-26T21:52:11,802 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/828b3f090c91:0 server-side Connection retries=45 2024-11-26T21:52:11,802 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-26T21:52:11,802 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-26T21:52:11,802 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-26T21:52:11,802 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-26T21:52:11,802 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-26T21:52:11,802 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-26T21:52:11,802 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-26T21:52:11,803 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:39841 2024-11-26T21:52:11,805 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:39841 connecting to ZooKeeper ensemble=127.0.0.1:53598 2024-11-26T21:52:11,805 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-26T21:52:11,805 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-26T21:52:11,808 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:398410x0, quorum=127.0.0.1:53598, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-26T21:52:11,808 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:39841-0x1007e0536a90003 connected 2024-11-26T21:52:11,808 DEBUG [Time-limited test {}] zookeeper.ZKUtil(111): regionserver:39841-0x1007e0536a90003, quorum=127.0.0.1:53598, baseZNode=/hbase Set watcher on existing znode=/hbase/running 2024-11-26T21:52:11,808 DEBUG [pool-297-thread-1 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: INIT 2024-11-26T21:52:11,809 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-26T21:52:11,810 INFO [Time-limited test {}] bucket.FileIOEngine(99): Allocating cache 400 MB, on the path:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/b34d8522-cb7c-4d9b-6a0e-fe88ad6cba6d/bucket.cache 2024-11-26T21:52:11,810 INFO [Time-limited test {}] bucket.BucketCache(364): Instantiating BucketCache with acceptableFactor: 0.95, minFactor: 0.85, extraFreeFactor: 0.1, singleFactor: 0.25, multiFactor: 0.5, memoryFactor: 0.25, useStrongRef: false 2024-11-26T21:52:11,810 INFO [Time-limited test {}] bucket.BucketCache(478): Starting BucketCachePersisterThread 2024-11-26T21:52:11,810 INFO [Time-limited test {}] bucket.BucketCachePersister(38): BucketCachePersister started with interval: 100 2024-11-26T21:52:11,810 INFO [Thread-199 {}] bucket.BucketCache(1530): Started retrieving bucket cache from file 2024-11-26T21:52:11,810 INFO [Time-limited test {}] bucket.BucketCache(403): Started bucket cache; ioengine=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/b34d8522-cb7c-4d9b-6a0e-fe88ad6cba6d/bucket.cache, capacity=400 MB, blockSize=64 KB, writerThreadNum=3, writerQLen=64, persistencePath=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/b34d8522-cb7c-4d9b-6a0e-fe88ad6cba6d/bucket.persistence, bucketAllocator=org.apache.hadoop.hbase.io.hfile.bucket.BucketAllocator 2024-11-26T21:52:11,811 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-26T21:52:11,811 INFO [Thread-199 {}] bucket.BucketCache(1552): Reading new chunked format of persistence. 2024-11-26T21:52:11,811 DEBUG [Time-limited test {}] zookeeper.ZKUtil(111): regionserver:39841-0x1007e0536a90003, quorum=127.0.0.1:53598, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-11-26T21:52:11,812 INFO [Thread-199 {}] bucket.BucketCache(1705): Retrieved 1 of chunks with blockCount = 1. 2024-11-26T21:52:11,813 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:39841-0x1007e0536a90003, quorum=127.0.0.1:53598, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-26T21:52:11,816 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=39841 2024-11-26T21:52:11,817 DEBUG [Thread-199 {}] bucket.BucketCache(1569): Updating region size map after retrieving cached file list 2024-11-26T21:52:11,817 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=39841 2024-11-26T21:52:11,817 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=39841 2024-11-26T21:52:11,817 INFO [Thread-199 {}] bucket.BucketAllocator(343): Cache totalSize=418148352, buckets=199, bucket capacity=2101248=(4*525312)=(FEWEST_ITEMS_IN_BUCKET*(largest configured bucketcache size)) 2024-11-26T21:52:11,817 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=39841 2024-11-26T21:52:11,817 INFO [Thread-199 {}] bucket.BucketCache(1562): Bucket cache retrieved from file successfully with size: 1 2024-11-26T21:52:11,817 INFO [Thread-199 {}] bucket.BucketCache(413): Persistent bucket cache recovery from /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/b34d8522-cb7c-4d9b-6a0e-fe88ad6cba6d/bucket.persistence is complete. 2024-11-26T21:52:11,817 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=39841 2024-11-26T21:52:11,819 INFO [RS:2;828b3f090c91:39841 {}] regionserver.HRegionServer(746): ClusterId : 45b5085c-0c14-4246-aa10-be0654098e00 2024-11-26T21:52:11,819 DEBUG [RS:2;828b3f090c91:39841 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-26T21:52:11,820 DEBUG [RS:2;828b3f090c91:39841 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-26T21:52:11,820 DEBUG [RS:2;828b3f090c91:39841 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-26T21:52:11,822 DEBUG [RS:2;828b3f090c91:39841 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-26T21:52:11,822 DEBUG [RS:2;828b3f090c91:39841 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@443be18f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=828b3f090c91/172.17.0.2:0 2024-11-26T21:52:11,833 DEBUG [RS:2;828b3f090c91:39841 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:2;828b3f090c91:39841 2024-11-26T21:52:11,833 INFO [RS:2;828b3f090c91:39841 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-26T21:52:11,833 INFO [RS:2;828b3f090c91:39841 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-26T21:52:11,833 DEBUG [RS:2;828b3f090c91:39841 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-26T21:52:11,834 INFO [RS:2;828b3f090c91:39841 {}] regionserver.HRegionServer(2659): reportForDuty to master=828b3f090c91,35155,1732657928779 with port=39841, startcode=1732657931801 2024-11-26T21:52:11,834 DEBUG [RS:2;828b3f090c91:39841 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-26T21:52:11,837 INFO [HMaster-EventLoopGroup-6-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:55945, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=RegionServerStatusService 2024-11-26T21:52:11,837 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=35155 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 828b3f090c91,39841,1732657931801 2024-11-26T21:52:11,837 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=35155 {}] master.ServerManager(517): Registering regionserver=828b3f090c91,39841,1732657931801 2024-11-26T21:52:11,839 DEBUG [RS:2;828b3f090c91:39841 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7430f913-2694-54cf-261c-654b69c5fe06/de866367-466a-3788-a91b-60c9e5aa6ab2 2024-11-26T21:52:11,839 DEBUG [RS:2;828b3f090c91:39841 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=file:/ 2024-11-26T21:52:11,839 DEBUG [RS:2;828b3f090c91:39841 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-26T21:52:11,840 INFO [RS_OPEN_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] handler.AssignRegionHandler(132): Open testBlockEvictionOnGracefulStop1732657929590,,1732657929595.282fdb3a3c29b93634ce02b236635dc2. 2024-11-26T21:52:11,840 DEBUG [RS_OPEN_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(7752): Opening region: {ENCODED => 282fdb3a3c29b93634ce02b236635dc2, NAME => 'testBlockEvictionOnGracefulStop1732657929590,,1732657929595.282fdb3a3c29b93634ce02b236635dc2.', STARTKEY => '', ENDKEY => ''} 2024-11-26T21:52:11,840 DEBUG [RS_OPEN_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testBlockEvictionOnGracefulStop1732657929590 282fdb3a3c29b93634ce02b236635dc2 2024-11-26T21:52:11,840 DEBUG [RS_OPEN_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(898): Instantiated testBlockEvictionOnGracefulStop1732657929590,,1732657929595.282fdb3a3c29b93634ce02b236635dc2.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-26T21:52:11,841 DEBUG [RS_OPEN_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(7794): checking encryption for 282fdb3a3c29b93634ce02b236635dc2 2024-11-26T21:52:11,841 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35155-0x1007e0536a90000, quorum=127.0.0.1:53598, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-26T21:52:11,841 DEBUG [RS_OPEN_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(7797): checking classloading for 282fdb3a3c29b93634ce02b236635dc2 2024-11-26T21:52:11,841 DEBUG [RS:2;828b3f090c91:39841 {}] zookeeper.ZKUtil(111): regionserver:39841-0x1007e0536a90003, quorum=127.0.0.1:53598, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/828b3f090c91,39841,1732657931801 2024-11-26T21:52:11,841 WARN [RS:2;828b3f090c91:39841 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-26T21:52:11,841 INFO [RS:2;828b3f090c91:39841 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-26T21:52:11,842 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [828b3f090c91,39841,1732657931801] 2024-11-26T21:52:11,842 DEBUG [RS:2;828b3f090c91:39841 {}] regionserver.HRegionServer(1793): logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7430f913-2694-54cf-261c-654b69c5fe06/de866367-466a-3788-a91b-60c9e5aa6ab2/WALs/828b3f090c91,39841,1732657931801 2024-11-26T21:52:11,842 INFO [StoreOpener-282fdb3a3c29b93634ce02b236635dc2-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family family of region 282fdb3a3c29b93634ce02b236635dc2 2024-11-26T21:52:11,842 INFO [StoreOpener-282fdb3a3c29b93634ce02b236635dc2-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 282fdb3a3c29b93634ce02b236635dc2 columnFamilyName family 2024-11-26T21:52:11,843 DEBUG [StoreOpener-282fdb3a3c29b93634ce02b236635dc2-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T21:52:11,844 DEBUG [StoreFileOpener-282fdb3a3c29b93634ce02b236635dc2-family-1 {}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7430f913-2694-54cf-261c-654b69c5fe06/de866367-466a-3788-a91b-60c9e5aa6ab2/data/default/testBlockEvictionOnGracefulStop1732657929590/282fdb3a3c29b93634ce02b236635dc2/family/2ae1dddb00634a5681fd7785f8291bc0, delay=931 ms 2024-11-26T21:52:11,845 DEBUG [StoreOpener-282fdb3a3c29b93634ce02b236635dc2-1 {}] regionserver.StoreEngine(278): loaded file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7430f913-2694-54cf-261c-654b69c5fe06/de866367-466a-3788-a91b-60c9e5aa6ab2/data/default/testBlockEvictionOnGracefulStop1732657929590/282fdb3a3c29b93634ce02b236635dc2/family/2ae1dddb00634a5681fd7785f8291bc0 2024-11-26T21:52:11,846 INFO [StoreOpener-282fdb3a3c29b93634ce02b236635dc2-1 {}] regionserver.HStore(327): Store=282fdb3a3c29b93634ce02b236635dc2/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-26T21:52:11,846 DEBUG [RS_OPEN_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1038): replaying wal for 282fdb3a3c29b93634ce02b236635dc2 2024-11-26T21:52:11,847 DEBUG [RS_OPEN_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7430f913-2694-54cf-261c-654b69c5fe06/de866367-466a-3788-a91b-60c9e5aa6ab2/data/default/testBlockEvictionOnGracefulStop1732657929590/282fdb3a3c29b93634ce02b236635dc2 2024-11-26T21:52:11,847 INFO [RS:2;828b3f090c91:39841 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-26T21:52:11,848 DEBUG [RS_OPEN_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7430f913-2694-54cf-261c-654b69c5fe06/de866367-466a-3788-a91b-60c9e5aa6ab2/data/default/testBlockEvictionOnGracefulStop1732657929590/282fdb3a3c29b93634ce02b236635dc2 2024-11-26T21:52:11,848 DEBUG [RS_OPEN_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1048): stopping wal replay for 282fdb3a3c29b93634ce02b236635dc2 2024-11-26T21:52:11,848 DEBUG [RS_OPEN_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1060): Cleaning up temporary data for 282fdb3a3c29b93634ce02b236635dc2 2024-11-26T21:52:11,849 DEBUG [RS_OPEN_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1093): writing seq id for 282fdb3a3c29b93634ce02b236635dc2 2024-11-26T21:52:11,850 INFO [RS:2;828b3f090c91:39841 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-26T21:52:11,850 INFO [RS_OPEN_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1114): Opened 282fdb3a3c29b93634ce02b236635dc2; next sequenceid=10; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=68034553, jitterRate=0.013793841004371643}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-26T21:52:11,850 DEBUG [RS_OPEN_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 282fdb3a3c29b93634ce02b236635dc2 2024-11-26T21:52:11,850 INFO [RS:2;828b3f090c91:39841 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-26T21:52:11,850 INFO [RS:2;828b3f090c91:39841 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-26T21:52:11,851 INFO [RS:2;828b3f090c91:39841 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-26T21:52:11,851 DEBUG [RS_OPEN_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1006): Region open journal for 282fdb3a3c29b93634ce02b236635dc2: Running coprocessor pre-open hook at 1732657931841Writing region info on filesystem at 1732657931841Initializing all the Stores at 1732657931841Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732657931841Cleaning up temporary data from old regions at 1732657931848 (+7 ms)Running coprocessor post-open hooks at 1732657931850 (+2 ms)Region opened successfully at 1732657931851 (+1 ms) 2024-11-26T21:52:11,851 INFO [RS:2;828b3f090c91:39841 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-26T21:52:11,851 INFO [RS:2;828b3f090c91:39841 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-26T21:52:11,851 DEBUG [RS:2;828b3f090c91:39841 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/828b3f090c91:0, corePoolSize=1, maxPoolSize=1 2024-11-26T21:52:11,851 DEBUG [RS:2;828b3f090c91:39841 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/828b3f090c91:0, corePoolSize=1, maxPoolSize=1 2024-11-26T21:52:11,851 DEBUG [RS:2;828b3f090c91:39841 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/828b3f090c91:0, corePoolSize=1, maxPoolSize=1 2024-11-26T21:52:11,851 DEBUG [RS:2;828b3f090c91:39841 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/828b3f090c91:0, corePoolSize=1, maxPoolSize=1 2024-11-26T21:52:11,852 DEBUG [RS:2;828b3f090c91:39841 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/828b3f090c91:0, corePoolSize=1, maxPoolSize=1 2024-11-26T21:52:11,852 INFO [RS_OPEN_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegionServer(2236): Post open deploy tasks for testBlockEvictionOnGracefulStop1732657929590,,1732657929595.282fdb3a3c29b93634ce02b236635dc2., pid=9, masterSystemTime=1732657931838 2024-11-26T21:52:11,852 DEBUG [RS:2;828b3f090c91:39841 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/828b3f090c91:0, corePoolSize=2, maxPoolSize=2 2024-11-26T21:52:11,852 DEBUG [RS:2;828b3f090c91:39841 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/828b3f090c91:0, corePoolSize=1, maxPoolSize=1 2024-11-26T21:52:11,852 DEBUG [RS:2;828b3f090c91:39841 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/828b3f090c91:0, corePoolSize=1, maxPoolSize=1 2024-11-26T21:52:11,852 DEBUG [RS:2;828b3f090c91:39841 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/828b3f090c91:0, corePoolSize=1, maxPoolSize=1 2024-11-26T21:52:11,852 DEBUG [RS:2;828b3f090c91:39841 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/828b3f090c91:0, corePoolSize=1, maxPoolSize=1 2024-11-26T21:52:11,852 DEBUG [RS:2;828b3f090c91:39841 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/828b3f090c91:0, corePoolSize=1, maxPoolSize=1 2024-11-26T21:52:11,852 DEBUG [RS:2;828b3f090c91:39841 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/828b3f090c91:0, corePoolSize=1, maxPoolSize=1 2024-11-26T21:52:11,852 DEBUG [RS:2;828b3f090c91:39841 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/828b3f090c91:0, corePoolSize=3, maxPoolSize=3 2024-11-26T21:52:11,852 DEBUG [RS:2;828b3f090c91:39841 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/828b3f090c91:0, corePoolSize=3, maxPoolSize=3 2024-11-26T21:52:11,853 INFO [RS:2;828b3f090c91:39841 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-26T21:52:11,853 INFO [RS:2;828b3f090c91:39841 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-26T21:52:11,853 INFO [RS:2;828b3f090c91:39841 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-26T21:52:11,853 INFO [RS:2;828b3f090c91:39841 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-26T21:52:11,853 INFO [RS:2;828b3f090c91:39841 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-26T21:52:11,853 INFO [RS:2;828b3f090c91:39841 {}] hbase.ChoreService(168): Chore ScheduledChore name=828b3f090c91,39841,1732657931801-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-26T21:52:11,854 DEBUG [RS_OPEN_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegionServer(2266): Finished post open deploy task for testBlockEvictionOnGracefulStop1732657929590,,1732657929595.282fdb3a3c29b93634ce02b236635dc2. 2024-11-26T21:52:11,854 INFO [RS_OPEN_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] handler.AssignRegionHandler(153): Opened testBlockEvictionOnGracefulStop1732657929590,,1732657929595.282fdb3a3c29b93634ce02b236635dc2. 2024-11-26T21:52:11,855 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=8 updating hbase:meta row=282fdb3a3c29b93634ce02b236635dc2, regionState=OPEN, openSeqNum=10, regionLocation=828b3f090c91,42409,1732657928858 2024-11-26T21:52:11,858 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-8-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=9, ppid=8, state=RUNNABLE, hasLock=false; OpenRegionProcedure 282fdb3a3c29b93634ce02b236635dc2, server=828b3f090c91,42409,1732657928858 because future has completed 2024-11-26T21:52:11,861 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=9, resume processing ppid=8 2024-11-26T21:52:11,861 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=9, ppid=8, state=SUCCESS, hasLock=false; OpenRegionProcedure 282fdb3a3c29b93634ce02b236635dc2, server=828b3f090c91,42409,1732657928858 in 175 msec 2024-11-26T21:52:11,863 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=8, resume processing ppid=7 2024-11-26T21:52:11,864 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=8, ppid=7, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1732657929590, region=282fdb3a3c29b93634ce02b236635dc2, ASSIGN in 338 msec 2024-11-26T21:52:11,864 INFO [PEWorker-2 {}] procedure.ServerCrashProcedure(291): removed crashed server 828b3f090c91,33857,1732657928830 after splitting done 2024-11-26T21:52:11,865 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=7, state=SUCCESS, hasLock=false; ServerCrashProcedure 828b3f090c91,33857,1732657928830, splitWal=true, meta=false in 354 msec 2024-11-26T21:52:11,870 INFO [RS:2;828b3f090c91:39841 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-26T21:52:11,870 INFO [RS:2;828b3f090c91:39841 {}] hbase.ChoreService(168): Chore ScheduledChore name=828b3f090c91,39841,1732657931801-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-26T21:52:11,870 INFO [RS:2;828b3f090c91:39841 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-26T21:52:11,870 INFO [RS:2;828b3f090c91:39841 {}] regionserver.Replication(171): 828b3f090c91,39841,1732657931801 started 2024-11-26T21:52:11,884 INFO [RS:2;828b3f090c91:39841 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-26T21:52:11,884 INFO [RS:2;828b3f090c91:39841 {}] regionserver.HRegionServer(1482): Serving as 828b3f090c91,39841,1732657931801, RpcServer on 828b3f090c91/172.17.0.2:39841, sessionid=0x1007e0536a90003 2024-11-26T21:52:11,884 DEBUG [RS:2;828b3f090c91:39841 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-26T21:52:11,885 DEBUG [RS:2;828b3f090c91:39841 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 828b3f090c91,39841,1732657931801 2024-11-26T21:52:11,885 DEBUG [RS:2;828b3f090c91:39841 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '828b3f090c91,39841,1732657931801' 2024-11-26T21:52:11,885 DEBUG [RS:2;828b3f090c91:39841 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-26T21:52:11,885 DEBUG [RS:2;828b3f090c91:39841 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-26T21:52:11,886 DEBUG [RS:2;828b3f090c91:39841 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-26T21:52:11,886 DEBUG [RS:2;828b3f090c91:39841 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-26T21:52:11,886 DEBUG [RS:2;828b3f090c91:39841 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 828b3f090c91,39841,1732657931801 2024-11-26T21:52:11,886 DEBUG [RS:2;828b3f090c91:39841 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '828b3f090c91,39841,1732657931801' 2024-11-26T21:52:11,886 DEBUG [RS:2;828b3f090c91:39841 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-26T21:52:11,887 DEBUG [RS:2;828b3f090c91:39841 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-26T21:52:11,887 DEBUG [RS:2;828b3f090c91:39841 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-26T21:52:11,887 INFO [RS:2;828b3f090c91:39841 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-26T21:52:11,887 INFO [RS:2;828b3f090c91:39841 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-26T21:52:11,988 INFO [RS:2;828b3f090c91:39841 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-26T21:52:11,989 INFO [RS:2;828b3f090c91:39841 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=828b3f090c91%2C39841%2C1732657931801, suffix=, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7430f913-2694-54cf-261c-654b69c5fe06/de866367-466a-3788-a91b-60c9e5aa6ab2/WALs/828b3f090c91,39841,1732657931801, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7430f913-2694-54cf-261c-654b69c5fe06/de866367-466a-3788-a91b-60c9e5aa6ab2/oldWALs, maxLogs=55 2024-11-26T21:52:12,006 INFO [RS:2;828b3f090c91:39841 {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7430f913-2694-54cf-261c-654b69c5fe06/de866367-466a-3788-a91b-60c9e5aa6ab2/WALs/828b3f090c91,39841,1732657931801/828b3f090c91%2C39841%2C1732657931801.1732657931989 2024-11-26T21:52:12,006 DEBUG [RS:2;828b3f090c91:39841 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-11-26T21:52:12,385 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster 2024-11-26T21:52:12,385 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-26T21:52:12,385 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79) at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611) at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.io.hfile.TestBlockEvictionOnRegionMovement.tearDown(TestBlockEvictionOnRegionMovement.java:174) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-26T21:52:12,385 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T21:52:12,386 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T21:52:12,386 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-26T21:52:12,386 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2024-11-26T21:52:12,386 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=1522859217, stopped=false 2024-11-26T21:52:12,386 INFO [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=828b3f090c91,35155,1732657928779 2024-11-26T21:52:12,389 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35155-0x1007e0536a90000, quorum=127.0.0.1:53598, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-26T21:52:12,389 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42409-0x1007e0536a90002, quorum=127.0.0.1:53598, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-26T21:52:12,389 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35155-0x1007e0536a90000, quorum=127.0.0.1:53598, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-26T21:52:12,389 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39841-0x1007e0536a90003, quorum=127.0.0.1:53598, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-26T21:52:12,389 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42409-0x1007e0536a90002, quorum=127.0.0.1:53598, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-26T21:52:12,389 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39841-0x1007e0536a90003, quorum=127.0.0.1:53598, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-26T21:52:12,390 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-26T21:52:12,391 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:35155-0x1007e0536a90000, quorum=127.0.0.1:53598, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-26T21:52:12,391 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:39841-0x1007e0536a90003, quorum=127.0.0.1:53598, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-26T21:52:12,391 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-26T21:52:12,392 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:42409-0x1007e0536a90002, quorum=127.0.0.1:53598, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-26T21:52:12,392 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277) at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265) at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416) at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.io.hfile.TestBlockEvictionOnRegionMovement.tearDown(TestBlockEvictionOnRegionMovement.java:174) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-26T21:52:12,392 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T21:52:12,393 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '828b3f090c91,42409,1732657928858' ***** 2024-11-26T21:52:12,393 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-11-26T21:52:12,393 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '828b3f090c91,39841,1732657931801' ***** 2024-11-26T21:52:12,394 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-11-26T21:52:12,394 INFO [RS:1;828b3f090c91:42409 {}] bucket.BucketCache(1738): Disabling cache 2024-11-26T21:52:12,394 INFO [Time-limited test-BucketCacheWriter-0 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-0 exiting, cacheEnabled=false 2024-11-26T21:52:12,394 INFO [Time-limited test-BucketCacheWriter-1 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-1 exiting, cacheEnabled=false 2024-11-26T21:52:12,394 INFO [RS:1;828b3f090c91:42409 {}] bucket.BucketCache(1762): Shutdown bucket cache: IO persistent=true; path to write=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/b34d8522-cb7c-4d9b-6a0e-fe88ad6cba6d/bucket.persistence 2024-11-26T21:52:12,394 INFO [RS:2;828b3f090c91:39841 {}] bucket.BucketCache(1738): Disabling cache 2024-11-26T21:52:12,394 INFO [Time-limited test-BucketCacheWriter-2 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-2 exiting, cacheEnabled=false 2024-11-26T21:52:12,395 INFO [RS:1;828b3f090c91:42409 {}] bucket.BucketCache(1768): Shutting down cache persister thread. 2024-11-26T21:52:12,395 INFO [RS:2;828b3f090c91:39841 {}] bucket.BucketCache(1762): Shutdown bucket cache: IO persistent=true; path to write=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/b34d8522-cb7c-4d9b-6a0e-fe88ad6cba6d/bucket.persistence 2024-11-26T21:52:12,395 INFO [Time-limited test-BucketCacheWriter-0 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-0 exiting, cacheEnabled=false 2024-11-26T21:52:12,395 INFO [Time-limited test-BucketCacheWriter-2 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-2 exiting, cacheEnabled=false 2024-11-26T21:52:12,395 INFO [Time-limited test-BucketCacheWriter-1 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-1 exiting, cacheEnabled=false 2024-11-26T21:52:12,395 WARN [bucket-cache-persister {}] bucket.BucketCachePersister(64): Interrupting BucketCachePersister thread. java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCachePersister.run(BucketCachePersister.java:45) ~[classes/:?] 2024-11-26T21:52:12,395 INFO [RS:2;828b3f090c91:39841 {}] bucket.BucketCache(1768): Shutting down cache persister thread. 2024-11-26T21:52:12,395 DEBUG [RS:1;828b3f090c91:42409 {}] bucket.BucketCache(1487): Thread RS:1;828b3f090c91:42409 started persisting bucket cache to file 2024-11-26T21:52:12,395 WARN [bucket-cache-persister {}] bucket.BucketCachePersister(64): Interrupting BucketCachePersister thread. java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCachePersister.run(BucketCachePersister.java:45) ~[classes/:?] 2024-11-26T21:52:12,395 DEBUG [RS:1;828b3f090c91:42409 {}] bucket.BucketCache(1494): Persist in new chunked persistence format. 2024-11-26T21:52:12,395 DEBUG [RS:1;828b3f090c91:42409 {}] bucket.BucketCache(1673): persistToFile: before persisting backing map size: 0, fullycachedFiles size: 0, chunkSize: 10000 2024-11-26T21:52:12,398 DEBUG [RS:1;828b3f090c91:42409 {}] bucket.BucketCache(1680): persistToFile: after persisting backing map size: 0, fullycachedFiles size: 0 2024-11-26T21:52:12,398 DEBUG [RS:1;828b3f090c91:42409 {}] bucket.BucketCache(1498): PersistToFile: after persisting backing map size: 0, fullycachedFiles size: 0, file name: bucket.persistence1732657932395 2024-11-26T21:52:12,398 DEBUG [RS:1;828b3f090c91:42409 {}] bucket.BucketCache(1509): Thread RS:1;828b3f090c91:42409 finished persisting bucket cache to file, renaming 2024-11-26T21:52:12,398 INFO [RS:1;828b3f090c91:42409 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-26T21:52:12,398 INFO [RS:1;828b3f090c91:42409 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-26T21:52:12,398 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-26T21:52:12,398 INFO [RS:1;828b3f090c91:42409 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-26T21:52:12,399 INFO [RS:1;828b3f090c91:42409 {}] regionserver.HRegionServer(3091): Received CLOSE for 282fdb3a3c29b93634ce02b236635dc2 2024-11-26T21:52:12,399 INFO [RS:1;828b3f090c91:42409 {}] regionserver.HRegionServer(959): stopping server 828b3f090c91,42409,1732657928858 2024-11-26T21:52:12,399 INFO [RS:1;828b3f090c91:42409 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-26T21:52:12,399 INFO [RS:1;828b3f090c91:42409 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:1;828b3f090c91:42409. 2024-11-26T21:52:12,399 INFO [RS_CLOSE_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1662): Closing region testBlockEvictionOnGracefulStop1732657929590,,1732657929595.282fdb3a3c29b93634ce02b236635dc2. during a graceful stop, and cache persistence is on, so setting evict on close to false. 2024-11-26T21:52:12,399 DEBUG [RS:1;828b3f090c91:42409 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-26T21:52:12,399 DEBUG [RS:1;828b3f090c91:42409 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T21:52:12,399 DEBUG [RS_CLOSE_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 282fdb3a3c29b93634ce02b236635dc2, disabling compactions & flushes 2024-11-26T21:52:12,399 INFO [RS_CLOSE_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testBlockEvictionOnGracefulStop1732657929590,,1732657929595.282fdb3a3c29b93634ce02b236635dc2. 2024-11-26T21:52:12,399 INFO [RS:1;828b3f090c91:42409 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-26T21:52:12,399 DEBUG [RS_CLOSE_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testBlockEvictionOnGracefulStop1732657929590,,1732657929595.282fdb3a3c29b93634ce02b236635dc2. 2024-11-26T21:52:12,399 INFO [RS:1;828b3f090c91:42409 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-26T21:52:12,399 INFO [RS:1;828b3f090c91:42409 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-26T21:52:12,399 DEBUG [RS_CLOSE_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testBlockEvictionOnGracefulStop1732657929590,,1732657929595.282fdb3a3c29b93634ce02b236635dc2. after waiting 0 ms 2024-11-26T21:52:12,399 INFO [RS:1;828b3f090c91:42409 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-11-26T21:52:12,399 DEBUG [RS_CLOSE_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testBlockEvictionOnGracefulStop1732657929590,,1732657929595.282fdb3a3c29b93634ce02b236635dc2. 2024-11-26T21:52:12,399 INFO [RS:1;828b3f090c91:42409 {}] regionserver.HRegionServer(1321): Waiting on 2 regions to close 2024-11-26T21:52:12,399 DEBUG [RS:1;828b3f090c91:42409 {}] regionserver.HRegionServer(1325): Online Regions={282fdb3a3c29b93634ce02b236635dc2=testBlockEvictionOnGracefulStop1732657929590,,1732657929595.282fdb3a3c29b93634ce02b236635dc2., 1588230740=hbase:meta,,1.1588230740} 2024-11-26T21:52:12,400 INFO [RS_CLOSE_META-regionserver/828b3f090c91:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1662): Closing region hbase:meta,,1.1588230740 during a graceful stop, and cache persistence is on, so setting evict on close to false. 2024-11-26T21:52:12,400 DEBUG [RS:1;828b3f090c91:42409 {}] regionserver.HRegionServer(1351): Waiting on 1588230740, 282fdb3a3c29b93634ce02b236635dc2 2024-11-26T21:52:12,400 DEBUG [RS_CLOSE_META-regionserver/828b3f090c91:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-26T21:52:12,400 INFO [RS_CLOSE_META-regionserver/828b3f090c91:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-26T21:52:12,400 DEBUG [RS_CLOSE_META-regionserver/828b3f090c91:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-26T21:52:12,400 DEBUG [RS_CLOSE_META-regionserver/828b3f090c91:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-26T21:52:12,400 DEBUG [RS_CLOSE_META-regionserver/828b3f090c91:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-26T21:52:12,400 DEBUG [StoreFileCloser-282fdb3a3c29b93634ce02b236635dc2-family-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7430f913-2694-54cf-261c-654b69c5fe06/de866367-466a-3788-a91b-60c9e5aa6ab2/data/default/testBlockEvictionOnGracefulStop1732657929590/282fdb3a3c29b93634ce02b236635dc2/family/2ae1dddb00634a5681fd7785f8291bc0 2024-11-26T21:52:12,400 INFO [RS_CLOSE_META-regionserver/828b3f090c91:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=2.96 KB heapSize=5.58 KB 2024-11-26T21:52:12,405 DEBUG [RS_CLOSE_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7430f913-2694-54cf-261c-654b69c5fe06/de866367-466a-3788-a91b-60c9e5aa6ab2/data/default/testBlockEvictionOnGracefulStop1732657929590/282fdb3a3c29b93634ce02b236635dc2/recovered.edits/12.seqid, newMaxSeqId=12, maxSeqId=9 2024-11-26T21:52:12,405 INFO [RS_CLOSE_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testBlockEvictionOnGracefulStop1732657929590,,1732657929595.282fdb3a3c29b93634ce02b236635dc2. 2024-11-26T21:52:12,405 DEBUG [RS:2;828b3f090c91:39841 {}] bucket.BucketCache(1487): Thread RS:2;828b3f090c91:39841 started persisting bucket cache to file 2024-11-26T21:52:12,405 DEBUG [RS_CLOSE_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 282fdb3a3c29b93634ce02b236635dc2: Waiting for close lock at 1732657932399Running coprocessor pre-close hooks at 1732657932399Disabling compacts and flushes for region at 1732657932399Disabling writes for close at 1732657932399Writing region close event to WAL at 1732657932400 (+1 ms)Running coprocessor post-close hooks at 1732657932405 (+5 ms)Closed at 1732657932405 2024-11-26T21:52:12,405 DEBUG [RS:2;828b3f090c91:39841 {}] bucket.BucketCache(1494): Persist in new chunked persistence format. 2024-11-26T21:52:12,405 DEBUG [RS:2;828b3f090c91:39841 {}] bucket.BucketCache(1673): persistToFile: before persisting backing map size: 1, fullycachedFiles size: 0, chunkSize: 10000 2024-11-26T21:52:12,405 DEBUG [RS_CLOSE_REGION-regionserver/828b3f090c91:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testBlockEvictionOnGracefulStop1732657929590,,1732657929595.282fdb3a3c29b93634ce02b236635dc2. 2024-11-26T21:52:12,408 DEBUG [RS:2;828b3f090c91:39841 {}] bucket.BucketCache(1680): persistToFile: after persisting backing map size: 1, fullycachedFiles size: 0 2024-11-26T21:52:12,408 DEBUG [RS:2;828b3f090c91:39841 {}] bucket.BucketCache(1498): PersistToFile: after persisting backing map size: 1, fullycachedFiles size: 0, file name: bucket.persistence1732657932405 2024-11-26T21:52:12,408 DEBUG [RS:2;828b3f090c91:39841 {}] bucket.BucketCache(1509): Thread RS:2;828b3f090c91:39841 finished persisting bucket cache to file, renaming 2024-11-26T21:52:12,409 INFO [RS:2;828b3f090c91:39841 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-26T21:52:12,409 INFO [RS:2;828b3f090c91:39841 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-26T21:52:12,409 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-26T21:52:12,409 INFO [RS:2;828b3f090c91:39841 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-26T21:52:12,409 INFO [RS:2;828b3f090c91:39841 {}] regionserver.HRegionServer(959): stopping server 828b3f090c91,39841,1732657931801 2024-11-26T21:52:12,409 INFO [RS:2;828b3f090c91:39841 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-26T21:52:12,409 INFO [RS:2;828b3f090c91:39841 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:2;828b3f090c91:39841. 2024-11-26T21:52:12,409 DEBUG [RS:2;828b3f090c91:39841 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-26T21:52:12,409 DEBUG [RS:2;828b3f090c91:39841 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T21:52:12,409 INFO [RS:2;828b3f090c91:39841 {}] regionserver.HRegionServer(976): stopping server 828b3f090c91,39841,1732657931801; all regions closed. 2024-11-26T21:52:12,410 DEBUG [RS_CLOSE_META-regionserver/828b3f090c91:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFile$WriterFactory(308): Unable to set drop behind on e7064f90383744c9988d5f6fbe62fb49 2024-11-26T21:52:12,411 DEBUG [RS:2;828b3f090c91:39841 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7430f913-2694-54cf-261c-654b69c5fe06/de866367-466a-3788-a91b-60c9e5aa6ab2/oldWALs 2024-11-26T21:52:12,411 INFO [RS:2;828b3f090c91:39841 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 828b3f090c91%2C39841%2C1732657931801:(num 1732657931989) 2024-11-26T21:52:12,411 DEBUG [RS:2;828b3f090c91:39841 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T21:52:12,411 INFO [RS:2;828b3f090c91:39841 {}] regionserver.LeaseManager(133): Closed leases 2024-11-26T21:52:12,411 INFO [RS:2;828b3f090c91:39841 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-26T21:52:12,412 INFO [RS:2;828b3f090c91:39841 {}] hbase.ChoreService(370): Chore service for: regionserver/828b3f090c91:0 had [ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-11-26T21:52:12,412 INFO [RS:2;828b3f090c91:39841 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-26T21:52:12,412 INFO [regionserver/828b3f090c91:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-26T21:52:12,412 INFO [RS:2;828b3f090c91:39841 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-26T21:52:12,412 INFO [RS:2;828b3f090c91:39841 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-26T21:52:12,412 INFO [RS:2;828b3f090c91:39841 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-26T21:52:12,412 INFO [RS:2;828b3f090c91:39841 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:39841 2024-11-26T21:52:12,414 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35155-0x1007e0536a90000, quorum=127.0.0.1:53598, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-26T21:52:12,414 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39841-0x1007e0536a90003, quorum=127.0.0.1:53598, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/828b3f090c91,39841,1732657931801 2024-11-26T21:52:12,414 INFO [RS:2;828b3f090c91:39841 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-26T21:52:12,414 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [828b3f090c91,39841,1732657931801] 2024-11-26T21:52:12,415 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/828b3f090c91,39841,1732657931801 already deleted, retry=false 2024-11-26T21:52:12,415 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; 828b3f090c91,39841,1732657931801 expired; onlineServers=1 2024-11-26T21:52:12,424 DEBUG [RS_CLOSE_META-regionserver/828b3f090c91:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7430f913-2694-54cf-261c-654b69c5fe06/de866367-466a-3788-a91b-60c9e5aa6ab2/data/hbase/meta/1588230740/.tmp/info/e7064f90383744c9988d5f6fbe62fb49 is 209, key is testBlockEvictionOnGracefulStop1732657929590,,1732657929595.282fdb3a3c29b93634ce02b236635dc2./info:regioninfo/1732657931855/Put/seqid=0 2024-11-26T21:52:12,425 INFO [RS_CLOSE_META-regionserver/828b3f090c91:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=2.74 KB at sequenceid=13 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7430f913-2694-54cf-261c-654b69c5fe06/de866367-466a-3788-a91b-60c9e5aa6ab2/data/hbase/meta/1588230740/.tmp/info/e7064f90383744c9988d5f6fbe62fb49 2024-11-26T21:52:12,432 DEBUG [RS_CLOSE_META-regionserver/828b3f090c91:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 46c8e68296f04604a8e2008eded33dda 2024-11-26T21:52:12,445 DEBUG [RS_CLOSE_META-regionserver/828b3f090c91:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7430f913-2694-54cf-261c-654b69c5fe06/de866367-466a-3788-a91b-60c9e5aa6ab2/data/hbase/meta/1588230740/.tmp/ns/46c8e68296f04604a8e2008eded33dda is 43, key is default/ns:d/1732657929553/Put/seqid=0 2024-11-26T21:52:12,446 INFO [RS_CLOSE_META-regionserver/828b3f090c91:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=74 B at sequenceid=13 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7430f913-2694-54cf-261c-654b69c5fe06/de866367-466a-3788-a91b-60c9e5aa6ab2/data/hbase/meta/1588230740/.tmp/ns/46c8e68296f04604a8e2008eded33dda 2024-11-26T21:52:12,454 DEBUG [RS_CLOSE_META-regionserver/828b3f090c91:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFile$WriterFactory(308): Unable to set drop behind on e09f6379834847f2be5ad61c8d39020b 2024-11-26T21:52:12,454 INFO [regionserver/828b3f090c91:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-26T21:52:12,457 INFO [regionserver/828b3f090c91:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-26T21:52:12,468 DEBUG [RS_CLOSE_META-regionserver/828b3f090c91:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7430f913-2694-54cf-261c-654b69c5fe06/de866367-466a-3788-a91b-60c9e5aa6ab2/data/hbase/meta/1588230740/.tmp/table/e09f6379834847f2be5ad61c8d39020b is 80, key is testBlockEvictionOnGracefulStop1732657929590/table:state/1732657929980/Put/seqid=0 2024-11-26T21:52:12,468 INFO [RS_CLOSE_META-regionserver/828b3f090c91:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=152 B at sequenceid=13 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7430f913-2694-54cf-261c-654b69c5fe06/de866367-466a-3788-a91b-60c9e5aa6ab2/data/hbase/meta/1588230740/.tmp/table/e09f6379834847f2be5ad61c8d39020b 2024-11-26T21:52:12,471 DEBUG [RS_CLOSE_META-regionserver/828b3f090c91:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7430f913-2694-54cf-261c-654b69c5fe06/de866367-466a-3788-a91b-60c9e5aa6ab2/data/hbase/meta/1588230740/.tmp/info/e7064f90383744c9988d5f6fbe62fb49 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7430f913-2694-54cf-261c-654b69c5fe06/de866367-466a-3788-a91b-60c9e5aa6ab2/data/hbase/meta/1588230740/info/e7064f90383744c9988d5f6fbe62fb49 2024-11-26T21:52:12,472 DEBUG [RS_CLOSE_META-regionserver/828b3f090c91:0-0 {event_type=M_RS_CLOSE_META}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7430f913-2694-54cf-261c-654b69c5fe06/de866367-466a-3788-a91b-60c9e5aa6ab2/data/hbase/meta/1588230740/info/e7064f90383744c9988d5f6fbe62fb49, delay=971 ms 2024-11-26T21:52:12,473 INFO [RS_CLOSE_META-regionserver/828b3f090c91:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7430f913-2694-54cf-261c-654b69c5fe06/de866367-466a-3788-a91b-60c9e5aa6ab2/data/hbase/meta/1588230740/info/e7064f90383744c9988d5f6fbe62fb49, entries=14, sequenceid=13, filesize=7.5 K 2024-11-26T21:52:12,474 DEBUG [RS_CLOSE_META-regionserver/828b3f090c91:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7430f913-2694-54cf-261c-654b69c5fe06/de866367-466a-3788-a91b-60c9e5aa6ab2/data/hbase/meta/1588230740/.tmp/ns/46c8e68296f04604a8e2008eded33dda as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7430f913-2694-54cf-261c-654b69c5fe06/de866367-466a-3788-a91b-60c9e5aa6ab2/data/hbase/meta/1588230740/ns/46c8e68296f04604a8e2008eded33dda 2024-11-26T21:52:12,475 DEBUG [RS_CLOSE_META-regionserver/828b3f090c91:0-0 {event_type=M_RS_CLOSE_META}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7430f913-2694-54cf-261c-654b69c5fe06/de866367-466a-3788-a91b-60c9e5aa6ab2/data/hbase/meta/1588230740/ns/46c8e68296f04604a8e2008eded33dda, delay=975 ms 2024-11-26T21:52:12,476 INFO [RS_CLOSE_META-regionserver/828b3f090c91:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7430f913-2694-54cf-261c-654b69c5fe06/de866367-466a-3788-a91b-60c9e5aa6ab2/data/hbase/meta/1588230740/ns/46c8e68296f04604a8e2008eded33dda, entries=2, sequenceid=13, filesize=5.0 K 2024-11-26T21:52:12,476 DEBUG [RS_CLOSE_META-regionserver/828b3f090c91:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7430f913-2694-54cf-261c-654b69c5fe06/de866367-466a-3788-a91b-60c9e5aa6ab2/data/hbase/meta/1588230740/.tmp/table/e09f6379834847f2be5ad61c8d39020b as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7430f913-2694-54cf-261c-654b69c5fe06/de866367-466a-3788-a91b-60c9e5aa6ab2/data/hbase/meta/1588230740/table/e09f6379834847f2be5ad61c8d39020b 2024-11-26T21:52:12,477 DEBUG [RS_CLOSE_META-regionserver/828b3f090c91:0-0 {event_type=M_RS_CLOSE_META}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7430f913-2694-54cf-261c-654b69c5fe06/de866367-466a-3788-a91b-60c9e5aa6ab2/data/hbase/meta/1588230740/table/e09f6379834847f2be5ad61c8d39020b, delay=944 ms 2024-11-26T21:52:12,479 INFO [RS_CLOSE_META-regionserver/828b3f090c91:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7430f913-2694-54cf-261c-654b69c5fe06/de866367-466a-3788-a91b-60c9e5aa6ab2/data/hbase/meta/1588230740/table/e09f6379834847f2be5ad61c8d39020b, entries=2, sequenceid=13, filesize=5.3 K 2024-11-26T21:52:12,480 INFO [RS_CLOSE_META-regionserver/828b3f090c91:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~2.96 KB/3035, heapSize ~5.28 KB/5408, currentSize=0 B/0 for 1588230740 in 80ms, sequenceid=13, compaction requested=false 2024-11-26T21:52:12,480 DEBUG [StoreFileCloser-1588230740-info-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7430f913-2694-54cf-261c-654b69c5fe06/de866367-466a-3788-a91b-60c9e5aa6ab2/data/hbase/meta/1588230740/info/e7064f90383744c9988d5f6fbe62fb49 2024-11-26T21:52:12,481 DEBUG [StoreFileCloser-1588230740-ns-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7430f913-2694-54cf-261c-654b69c5fe06/de866367-466a-3788-a91b-60c9e5aa6ab2/data/hbase/meta/1588230740/ns/46c8e68296f04604a8e2008eded33dda 2024-11-26T21:52:12,482 DEBUG [StoreFileCloser-1588230740-table-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7430f913-2694-54cf-261c-654b69c5fe06/de866367-466a-3788-a91b-60c9e5aa6ab2/data/hbase/meta/1588230740/table/e09f6379834847f2be5ad61c8d39020b 2024-11-26T21:52:12,486 DEBUG [RS_CLOSE_META-regionserver/828b3f090c91:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7430f913-2694-54cf-261c-654b69c5fe06/de866367-466a-3788-a91b-60c9e5aa6ab2/data/hbase/meta/1588230740/recovered.edits/16.seqid, newMaxSeqId=16, maxSeqId=1 2024-11-26T21:52:12,487 DEBUG [RS_CLOSE_META-regionserver/828b3f090c91:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-26T21:52:12,487 INFO [RS_CLOSE_META-regionserver/828b3f090c91:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-26T21:52:12,487 DEBUG [RS_CLOSE_META-regionserver/828b3f090c91:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1732657932400Running coprocessor pre-close hooks at 1732657932400Disabling compacts and flushes for region at 1732657932400Disabling writes for close at 1732657932400Obtaining lock to block concurrent updates at 1732657932400Preparing flush snapshotting stores in 1588230740 at 1732657932400Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=3035, getHeapSize=5648, getOffHeapSize=0, getCellsCount=22 at 1732657932400Flushing stores of hbase:meta,,1.1588230740 at 1732657932401 (+1 ms)Flushing 1588230740/info: creating writer at 1732657932401Flushing 1588230740/info: appending metadata at 1732657932424 (+23 ms)Flushing 1588230740/info: closing flushed file at 1732657932424Flushing 1588230740/ns: creating writer at 1732657932427 (+3 ms)Flushing 1588230740/ns: appending metadata at 1732657932445 (+18 ms)Flushing 1588230740/ns: closing flushed file at 1732657932445Flushing 1588230740/table: creating writer at 1732657932448 (+3 ms)Flushing 1588230740/table: appending metadata at 1732657932467 (+19 ms)Flushing 1588230740/table: closing flushed file at 1732657932467Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@67135fe6: reopening flushed file at 1732657932471 (+4 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@3210cfd: reopening flushed file at 1732657932473 (+2 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@561ec6e7: reopening flushed file at 1732657932476 (+3 ms)Finished flush of dataSize ~2.96 KB/3035, heapSize ~5.28 KB/5408, currentSize=0 B/0 for 1588230740 in 80ms, sequenceid=13, compaction requested=false at 1732657932480 (+4 ms)Writing region close event to WAL at 1732657932482 (+2 ms)Running coprocessor post-close hooks at 1732657932486 (+4 ms)Closed at 1732657932487 (+1 ms) 2024-11-26T21:52:12,487 DEBUG [RS_CLOSE_META-regionserver/828b3f090c91:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-11-26T21:52:12,515 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39841-0x1007e0536a90003, quorum=127.0.0.1:53598, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-26T21:52:12,515 INFO [RS:2;828b3f090c91:39841 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-26T21:52:12,515 INFO [RS:2;828b3f090c91:39841 {}] regionserver.HRegionServer(1031): Exiting; stopping=828b3f090c91,39841,1732657931801; zookeeper connection closed. 2024-11-26T21:52:12,515 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39841-0x1007e0536a90003, quorum=127.0.0.1:53598, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-26T21:52:12,516 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@768cc42e {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@768cc42e 2024-11-26T21:52:12,600 INFO [RS:1;828b3f090c91:42409 {}] regionserver.HRegionServer(976): stopping server 828b3f090c91,42409,1732657928858; all regions closed. 2024-11-26T21:52:12,604 DEBUG [RS:1;828b3f090c91:42409 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7430f913-2694-54cf-261c-654b69c5fe06/de866367-466a-3788-a91b-60c9e5aa6ab2/oldWALs 2024-11-26T21:52:12,604 INFO [RS:1;828b3f090c91:42409 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 828b3f090c91%2C42409%2C1732657928858.meta:.meta(num 1732657929502) 2024-11-26T21:52:12,607 DEBUG [RS:1;828b3f090c91:42409 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7430f913-2694-54cf-261c-654b69c5fe06/de866367-466a-3788-a91b-60c9e5aa6ab2/oldWALs 2024-11-26T21:52:12,607 INFO [RS:1;828b3f090c91:42409 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 828b3f090c91%2C42409%2C1732657928858:(num 1732657929301) 2024-11-26T21:52:12,607 DEBUG [RS:1;828b3f090c91:42409 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T21:52:12,607 INFO [RS:1;828b3f090c91:42409 {}] regionserver.LeaseManager(133): Closed leases 2024-11-26T21:52:12,607 INFO [RS:1;828b3f090c91:42409 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-26T21:52:12,607 INFO [RS:1;828b3f090c91:42409 {}] hbase.ChoreService(370): Chore service for: regionserver/828b3f090c91:0 had [ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-11-26T21:52:12,608 INFO [RS:1;828b3f090c91:42409 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-26T21:52:12,608 INFO [regionserver/828b3f090c91:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-26T21:52:12,608 INFO [RS:1;828b3f090c91:42409 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:42409 2024-11-26T21:52:12,610 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35155-0x1007e0536a90000, quorum=127.0.0.1:53598, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-26T21:52:12,610 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42409-0x1007e0536a90002, quorum=127.0.0.1:53598, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/828b3f090c91,42409,1732657928858 2024-11-26T21:52:12,610 INFO [RS:1;828b3f090c91:42409 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-26T21:52:12,611 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [828b3f090c91,42409,1732657928858] 2024-11-26T21:52:12,612 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/828b3f090c91,42409,1732657928858 already deleted, retry=false 2024-11-26T21:52:12,612 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; 828b3f090c91,42409,1732657928858 expired; onlineServers=0 2024-11-26T21:52:12,612 INFO [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master '828b3f090c91,35155,1732657928779' ***** 2024-11-26T21:52:12,612 INFO [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0 2024-11-26T21:52:12,612 INFO [M:0;828b3f090c91:35155 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-26T21:52:12,612 INFO [M:0;828b3f090c91:35155 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-26T21:52:12,612 DEBUG [M:0;828b3f090c91:35155 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2024-11-26T21:52:12,613 DEBUG [M:0;828b3f090c91:35155 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2024-11-26T21:52:12,613 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2024-11-26T21:52:12,613 DEBUG [master/828b3f090c91:0:becomeActiveMaster-HFileCleaner.small.0-1732657929069 {}] cleaner.HFileCleaner(306): Exit Thread[master/828b3f090c91:0:becomeActiveMaster-HFileCleaner.small.0-1732657929069,5,FailOnTimeoutGroup] 2024-11-26T21:52:12,613 DEBUG [master/828b3f090c91:0:becomeActiveMaster-HFileCleaner.large.0-1732657929069 {}] cleaner.HFileCleaner(306): Exit Thread[master/828b3f090c91:0:becomeActiveMaster-HFileCleaner.large.0-1732657929069,5,FailOnTimeoutGroup] 2024-11-26T21:52:12,613 INFO [M:0;828b3f090c91:35155 {}] hbase.ChoreService(370): Chore service for: master/828b3f090c91:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown 2024-11-26T21:52:12,613 INFO [M:0;828b3f090c91:35155 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-26T21:52:12,613 DEBUG [M:0;828b3f090c91:35155 {}] master.HMaster(1795): Stopping service threads 2024-11-26T21:52:12,613 INFO [M:0;828b3f090c91:35155 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2024-11-26T21:52:12,613 INFO [M:0;828b3f090c91:35155 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-26T21:52:12,613 INFO [M:0;828b3f090c91:35155 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-26T21:52:12,613 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2024-11-26T21:52:12,614 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35155-0x1007e0536a90000, quorum=127.0.0.1:53598, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-11-26T21:52:12,614 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35155-0x1007e0536a90000, quorum=127.0.0.1:53598, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-26T21:52:12,614 DEBUG [M:0;828b3f090c91:35155 {}] zookeeper.ZKUtil(347): master:35155-0x1007e0536a90000, quorum=127.0.0.1:53598, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error) 2024-11-26T21:52:12,614 WARN [M:0;828b3f090c91:35155 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null 2024-11-26T21:52:12,614 INFO [M:0;828b3f090c91:35155 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7430f913-2694-54cf-261c-654b69c5fe06/de866367-466a-3788-a91b-60c9e5aa6ab2/.lastflushedseqids 2024-11-26T21:52:12,620 INFO [M:0;828b3f090c91:35155 {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-26T21:52:12,620 INFO [M:0;828b3f090c91:35155 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2024-11-26T21:52:12,620 DEBUG [M:0;828b3f090c91:35155 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-26T21:52:12,620 INFO [M:0;828b3f090c91:35155 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-26T21:52:12,620 DEBUG [M:0;828b3f090c91:35155 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-26T21:52:12,620 DEBUG [M:0;828b3f090c91:35155 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-26T21:52:12,620 DEBUG [M:0;828b3f090c91:35155 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-26T21:52:12,620 INFO [M:0;828b3f090c91:35155 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=32.83 KB heapSize=41.08 KB 2024-11-26T21:52:12,628 DEBUG [M:0;828b3f090c91:35155 {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on db76cc9031824bd9b6d6a9f502bb7a77 2024-11-26T21:52:12,642 DEBUG [M:0;828b3f090c91:35155 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7430f913-2694-54cf-261c-654b69c5fe06/de866367-466a-3788-a91b-60c9e5aa6ab2/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/db76cc9031824bd9b6d6a9f502bb7a77 is 82, key is hbase:meta,,1/info:regioninfo/1732657929538/Put/seqid=0 2024-11-26T21:52:12,642 INFO [M:0;828b3f090c91:35155 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=504 B at sequenceid=83 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7430f913-2694-54cf-261c-654b69c5fe06/de866367-466a-3788-a91b-60c9e5aa6ab2/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/db76cc9031824bd9b6d6a9f502bb7a77 2024-11-26T21:52:12,649 DEBUG [M:0;828b3f090c91:35155 {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 24b599eabc20465cb91dc958985d5bc4 2024-11-26T21:52:12,661 DEBUG [M:0;828b3f090c91:35155 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7430f913-2694-54cf-261c-654b69c5fe06/de866367-466a-3788-a91b-60c9e5aa6ab2/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/24b599eabc20465cb91dc958985d5bc4 is 782, key is \x00\x00\x00\x00\x00\x00\x00\x04/proc:d/1732657929984/Put/seqid=0 2024-11-26T21:52:12,662 INFO [M:0;828b3f090c91:35155 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=32.09 KB at sequenceid=83 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7430f913-2694-54cf-261c-654b69c5fe06/de866367-466a-3788-a91b-60c9e5aa6ab2/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/24b599eabc20465cb91dc958985d5bc4 2024-11-26T21:52:12,668 DEBUG [M:0;828b3f090c91:35155 {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on aea8690449934fcbba0bf8a55b8cf4f6 2024-11-26T21:52:12,681 DEBUG [M:0;828b3f090c91:35155 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7430f913-2694-54cf-261c-654b69c5fe06/de866367-466a-3788-a91b-60c9e5aa6ab2/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/aea8690449934fcbba0bf8a55b8cf4f6 is 69, key is 828b3f090c91,39841,1732657931801/rs:state/1732657931838/Put/seqid=0 2024-11-26T21:52:12,681 INFO [M:0;828b3f090c91:35155 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=249 B at sequenceid=83 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7430f913-2694-54cf-261c-654b69c5fe06/de866367-466a-3788-a91b-60c9e5aa6ab2/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/aea8690449934fcbba0bf8a55b8cf4f6 2024-11-26T21:52:12,683 INFO [M:0;828b3f090c91:35155 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for aea8690449934fcbba0bf8a55b8cf4f6 2024-11-26T21:52:12,683 DEBUG [M:0;828b3f090c91:35155 {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7430f913-2694-54cf-261c-654b69c5fe06/de866367-466a-3788-a91b-60c9e5aa6ab2/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/db76cc9031824bd9b6d6a9f502bb7a77 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7430f913-2694-54cf-261c-654b69c5fe06/de866367-466a-3788-a91b-60c9e5aa6ab2/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/db76cc9031824bd9b6d6a9f502bb7a77 2024-11-26T21:52:12,685 INFO [M:0;828b3f090c91:35155 {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7430f913-2694-54cf-261c-654b69c5fe06/de866367-466a-3788-a91b-60c9e5aa6ab2/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/db76cc9031824bd9b6d6a9f502bb7a77, entries=8, sequenceid=83, filesize=5.5 K 2024-11-26T21:52:12,685 DEBUG [M:0;828b3f090c91:35155 {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7430f913-2694-54cf-261c-654b69c5fe06/de866367-466a-3788-a91b-60c9e5aa6ab2/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/24b599eabc20465cb91dc958985d5bc4 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7430f913-2694-54cf-261c-654b69c5fe06/de866367-466a-3788-a91b-60c9e5aa6ab2/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/24b599eabc20465cb91dc958985d5bc4 2024-11-26T21:52:12,687 INFO [M:0;828b3f090c91:35155 {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7430f913-2694-54cf-261c-654b69c5fe06/de866367-466a-3788-a91b-60c9e5aa6ab2/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/24b599eabc20465cb91dc958985d5bc4, entries=9, sequenceid=83, filesize=6.4 K 2024-11-26T21:52:12,687 DEBUG [M:0;828b3f090c91:35155 {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7430f913-2694-54cf-261c-654b69c5fe06/de866367-466a-3788-a91b-60c9e5aa6ab2/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/aea8690449934fcbba0bf8a55b8cf4f6 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7430f913-2694-54cf-261c-654b69c5fe06/de866367-466a-3788-a91b-60c9e5aa6ab2/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/aea8690449934fcbba0bf8a55b8cf4f6 2024-11-26T21:52:12,688 INFO [M:0;828b3f090c91:35155 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for aea8690449934fcbba0bf8a55b8cf4f6 2024-11-26T21:52:12,688 INFO [M:0;828b3f090c91:35155 {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7430f913-2694-54cf-261c-654b69c5fe06/de866367-466a-3788-a91b-60c9e5aa6ab2/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/aea8690449934fcbba0bf8a55b8cf4f6, entries=3, sequenceid=83, filesize=5.3 K 2024-11-26T21:52:12,689 INFO [M:0;828b3f090c91:35155 {}] regionserver.HRegion(3140): Finished flush of dataSize ~32.83 KB/33614, heapSize ~40.78 KB/41760, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 69ms, sequenceid=83, compaction requested=false 2024-11-26T21:52:12,690 INFO [M:0;828b3f090c91:35155 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-26T21:52:12,691 DEBUG [M:0;828b3f090c91:35155 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1732657932620Disabling compacts and flushes for region at 1732657932620Disabling writes for close at 1732657932620Obtaining lock to block concurrent updates at 1732657932620Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1732657932620Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=33614, getHeapSize=42000, getOffHeapSize=0, getCellsCount=98 at 1732657932621 (+1 ms)Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1732657932621Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1732657932621Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1732657932641 (+20 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1732657932641Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1732657932644 (+3 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1732657932661 (+17 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1732657932661Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1732657932663 (+2 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1732657932680 (+17 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1732657932680Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@28a8796e: reopening flushed file at 1732657932683 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@336cd294: reopening flushed file at 1732657932685 (+2 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@34f41a3f: reopening flushed file at 1732657932687 (+2 ms)Finished flush of dataSize ~32.83 KB/33614, heapSize ~40.78 KB/41760, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 69ms, sequenceid=83, compaction requested=false at 1732657932689 (+2 ms)Writing region close event to WAL at 1732657932690 (+1 ms)Closed at 1732657932690 2024-11-26T21:52:12,692 INFO [M:0;828b3f090c91:35155 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down. 2024-11-26T21:52:12,692 INFO [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-26T21:52:12,692 INFO [M:0;828b3f090c91:35155 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:35155 2024-11-26T21:52:12,692 INFO [M:0;828b3f090c91:35155 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-26T21:52:12,711 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42409-0x1007e0536a90002, quorum=127.0.0.1:53598, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-26T21:52:12,711 INFO [RS:1;828b3f090c91:42409 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-26T21:52:12,711 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42409-0x1007e0536a90002, quorum=127.0.0.1:53598, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-26T21:52:12,711 INFO [RS:1;828b3f090c91:42409 {}] regionserver.HRegionServer(1031): Exiting; stopping=828b3f090c91,42409,1732657928858; zookeeper connection closed. 2024-11-26T21:52:12,712 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@59ef99a1 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@59ef99a1 2024-11-26T21:52:12,712 INFO [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 3 regionserver(s) complete 2024-11-26T21:52:12,794 INFO [M:0;828b3f090c91:35155 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-26T21:52:12,794 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35155-0x1007e0536a90000, quorum=127.0.0.1:53598, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-26T21:52:12,794 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35155-0x1007e0536a90000, quorum=127.0.0.1:53598, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-26T21:52:12,800 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers 2024-11-26T21:52:12,802 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down 2024-11-26T21:52:12,803 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-26T21:52:12,809 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: io.hfile.TestBlockEvictionOnRegionMovement#testBlockEvictionOnGracefulStop Thread=71 (was 50) Potentially hanging thread: HMaster-EventLoopGroup-6-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-7-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-7 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HMaster-EventLoopGroup-6-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-6 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-5 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-8-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HMaster-EventLoopGroup-6-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-7-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-9-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-9-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-8-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-9 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-8 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-8-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-7-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=382 (was 346) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=137 (was 137), ProcessCount=11 (was 11), AvailableMemoryMB=10844 (was 10931)