2024-11-24 16:45:12,075 main DEBUG Apache Log4j Core 2.17.2 initializing configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@15a34df2 2024-11-24 16:45:12,094 main DEBUG Took 0.016348 seconds to load 1 plugins from package org.apache.hadoop.hbase.logging 2024-11-24 16:45:12,094 main DEBUG PluginManager 'Core' found 129 plugins 2024-11-24 16:45:12,095 main DEBUG PluginManager 'Level' found 0 plugins 2024-11-24 16:45:12,096 main DEBUG PluginManager 'Lookup' found 16 plugins 2024-11-24 16:45:12,097 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-24 16:45:12,104 main DEBUG PluginManager 'TypeConverter' found 26 plugins 2024-11-24 16:45:12,117 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.util.MBeans", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-24 16:45:12,119 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-24 16:45:12,120 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.logging.TestJul2Slf4j", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-24 16:45:12,120 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-24 16:45:12,121 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.zookeeper", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-24 16:45:12,122 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-24 16:45:12,123 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSinkAdapter", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-24 16:45:12,123 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-24 16:45:12,124 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSystemImpl", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-24 16:45:12,124 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-24 16:45:12,125 main DEBUG LoggerConfig$Builder(additivity="false", level="WARN", levelAndRefs="null", name="org.apache.directory", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-24 16:45:12,126 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-24 16:45:12,127 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.ipc.FailedServers", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-24 16:45:12,127 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-24 16:45:12,128 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsConfig", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-24 16:45:12,128 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-24 16:45:12,129 main DEBUG LoggerConfig$Builder(additivity="null", level="INFO", levelAndRefs="null", name="org.apache.hadoop.hbase.ScheduledChore", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-24 16:45:12,129 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-24 16:45:12,130 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.regionserver.RSRpcServices", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-24 16:45:12,130 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-24 16:45:12,131 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-24 16:45:12,131 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-24 16:45:12,132 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-24 16:45:12,133 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-24 16:45:12,133 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hbase.thirdparty.io.netty.channel", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-24 16:45:12,133 main DEBUG Building Plugin[name=root, class=org.apache.logging.log4j.core.config.LoggerConfig$RootLogger]. 2024-11-24 16:45:12,135 main DEBUG LoggerConfig$RootLogger$Builder(additivity="null", level="null", levelAndRefs="INFO,Console", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-24 16:45:12,137 main DEBUG Building Plugin[name=loggers, class=org.apache.logging.log4j.core.config.LoggersPlugin]. 2024-11-24 16:45:12,139 main DEBUG createLoggers(={org.apache.hadoop.metrics2.util.MBeans, org.apache.hadoop.hbase.logging.TestJul2Slf4j, org.apache.zookeeper, org.apache.hadoop.metrics2.impl.MetricsSinkAdapter, org.apache.hadoop.metrics2.impl.MetricsSystemImpl, org.apache.directory, org.apache.hadoop.hbase.ipc.FailedServers, org.apache.hadoop.metrics2.impl.MetricsConfig, org.apache.hadoop.hbase.ScheduledChore, org.apache.hadoop.hbase.regionserver.RSRpcServices, org.apache.hadoop, org.apache.hadoop.hbase, org.apache.hbase.thirdparty.io.netty.channel, root}) 2024-11-24 16:45:12,140 main DEBUG Building Plugin[name=layout, class=org.apache.logging.log4j.core.layout.PatternLayout]. 2024-11-24 16:45:12,142 main DEBUG PatternLayout$Builder(pattern="%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n", PatternSelector=null, Configuration(PropertiesConfig), Replace=null, charset="null", alwaysWriteExceptions="null", disableAnsi="null", noConsoleNoAnsi="null", header="null", footer="null") 2024-11-24 16:45:12,142 main DEBUG PluginManager 'Converter' found 47 plugins 2024-11-24 16:45:12,154 main DEBUG Building Plugin[name=appender, class=org.apache.hadoop.hbase.logging.HBaseTestAppender]. 2024-11-24 16:45:12,158 main DEBUG HBaseTestAppender$Builder(target="SYSTEM_ERR", maxSize="1G", bufferedIo="null", bufferSize="null", immediateFlush="null", ignoreExceptions="null", PatternLayout(%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n), name="Console", Configuration(PropertiesConfig), Filter=null, ={}) 2024-11-24 16:45:12,160 main DEBUG Starting HBaseTestOutputStreamManager SYSTEM_ERR 2024-11-24 16:45:12,160 main DEBUG Building Plugin[name=appenders, class=org.apache.logging.log4j.core.config.AppendersPlugin]. 2024-11-24 16:45:12,161 main DEBUG createAppenders(={Console}) 2024-11-24 16:45:12,161 main DEBUG Configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@15a34df2 initialized 2024-11-24 16:45:12,162 main DEBUG Starting configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@15a34df2 2024-11-24 16:45:12,162 main DEBUG Started configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@15a34df2 OK. 2024-11-24 16:45:12,162 main DEBUG Shutting down OutputStreamManager SYSTEM_OUT.false.false-1 2024-11-24 16:45:12,163 main DEBUG OutputStream closed 2024-11-24 16:45:12,163 main DEBUG Shut down OutputStreamManager SYSTEM_OUT.false.false-1, all resources released: true 2024-11-24 16:45:12,163 main DEBUG Appender DefaultConsole-1 stopped with status true 2024-11-24 16:45:12,163 main DEBUG Stopped org.apache.logging.log4j.core.config.DefaultConfiguration@4f2b503c OK 2024-11-24 16:45:12,229 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6 2024-11-24 16:45:12,230 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=StatusLogger 2024-11-24 16:45:12,231 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=ContextSelector 2024-11-24 16:45:12,232 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name= 2024-11-24 16:45:12,233 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.directory 2024-11-24 16:45:12,233 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSinkAdapter 2024-11-24 16:45:12,234 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.zookeeper 2024-11-24 16:45:12,234 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.logging.TestJul2Slf4j 2024-11-24 16:45:12,234 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSystemImpl 2024-11-24 16:45:12,235 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.util.MBeans 2024-11-24 16:45:12,235 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase 2024-11-24 16:45:12,235 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop 2024-11-24 16:45:12,235 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ipc.FailedServers 2024-11-24 16:45:12,236 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.regionserver.RSRpcServices 2024-11-24 16:45:12,236 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsConfig 2024-11-24 16:45:12,236 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hbase.thirdparty.io.netty.channel 2024-11-24 16:45:12,236 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ScheduledChore 2024-11-24 16:45:12,237 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Appenders,name=Console 2024-11-24 16:45:12,239 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-24 16:45:12,240 main DEBUG Reconfiguration complete for context[name=1dbd16a6] at URI jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-logging/target/hbase-logging-4.0.0-alpha-1-SNAPSHOT-tests.jar!/log4j2.properties (org.apache.logging.log4j.core.LoggerContext@31c7528f) with optional ClassLoader: null 2024-11-24 16:45:12,240 main DEBUG Shutdown hook enabled. Registering a new one. 2024-11-24 16:45:12,241 main DEBUG LoggerContext[name=1dbd16a6, org.apache.logging.log4j.core.LoggerContext@31c7528f] started OK. 2024-11-24T16:45:12,255 INFO [main {}] hbase.HBaseClassTestRule(94): Test class org.apache.hadoop.hbase.io.hfile.TestBlockEvictionOnRegionMovement timeout: 13 mins 2024-11-24 16:45:12,257 main DEBUG AsyncLogger.ThreadNameStrategy=UNCACHED (user specified null, default is UNCACHED) 2024-11-24 16:45:12,258 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-24T16:45:12,537 DEBUG [main {}] hbase.HBaseTestingUtil(323): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9797cf4c-7cf0-a873-8ba7-f30267b4a2f7 2024-11-24T16:45:12,561 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: io.hfile.TestBlockEvictionOnRegionMovement#testBlockEvictionOnRegionMove Thread=12, OpenFileDescriptor=287, MaxFileDescriptor=1048576, SystemLoadAverage=714, ProcessCount=11, AvailableMemoryMB=6951 2024-11-24T16:45:12,662 WARN [Time-limited test {}] util.NativeCodeLoader(60): Unable to load native-hadoop library for your platform... using builtin-java classes where applicable 2024-11-24T16:45:12,747 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-24T16:45:12,762 INFO [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9797cf4c-7cf0-a873-8ba7-f30267b4a2f7/cluster_d9e9c3c5-a255-d24e-9ae2-5a8d75af04af, deleteOnExit=true 2024-11-24T16:45:12,848 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9797cf4c-7cf0-a873-8ba7-f30267b4a2f7/cluster_d9e9c3c5-a255-d24e-9ae2-5a8d75af04af/zookeeper_0, clientPort=60963, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9797cf4c-7cf0-a873-8ba7-f30267b4a2f7/cluster_d9e9c3c5-a255-d24e-9ae2-5a8d75af04af/zookeeper_0/version-2, dataDirSize=457 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9797cf4c-7cf0-a873-8ba7-f30267b4a2f7/cluster_d9e9c3c5-a255-d24e-9ae2-5a8d75af04af/zookeeper_0/version-2, dataLogSize=457 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0 2024-11-24T16:45:12,860 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=60963 2024-11-24T16:45:12,860 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-24T16:45:12,944 INFO [Time-limited test {}] util.FSUtils(489): Created version file at /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9797cf4c-7cf0-a873-8ba7-f30267b4a2f7/39f4fdd4-f188-1d18-4286-5b0c4dc27cad with version=8 2024-11-24T16:45:12,944 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1139): Setting hbase.fs.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9797cf4c-7cf0-a873-8ba7-f30267b4a2f7/39f4fdd4-f188-1d18-4286-5b0c4dc27cad/hbase-staging 2024-11-24T16:45:13,043 DEBUG [Time-limited test {}] channel.MultithreadEventLoopGroup(44): -Dio.netty.eventLoopThreads: 16 2024-11-24T16:45:13,314 INFO [Time-limited test {}] client.ConnectionUtils(128): master/e335232f7491:0 server-side Connection retries=45 2024-11-24T16:45:13,325 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-24T16:45:13,326 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-24T16:45:13,334 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-24T16:45:13,334 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-24T16:45:13,335 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-24T16:45:13,481 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService 2024-11-24T16:45:13,577 WARN [Time-limited test {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-24T16:45:13,616 INFO [Time-limited test {}] metrics.MetricRegistriesLoader(60): Loaded MetricRegistries class org.apache.hadoop.hbase.metrics.impl.MetricRegistriesImpl 2024-11-24T16:45:13,626 DEBUG [Time-limited test {}] util.ClassSize(228): Using Unsafe to estimate memory layout 2024-11-24T16:45:13,630 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-24T16:45:13,673 DEBUG [Time-limited test {}] channel.DefaultChannelId(84): -Dio.netty.processId: 15392 (auto-detected) 2024-11-24T16:45:13,674 DEBUG [Time-limited test {}] channel.DefaultChannelId(106): -Dio.netty.machineId: 02:42:ac:ff:fe:11:00:02 (auto-detected) 2024-11-24T16:45:13,697 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:45629 2024-11-24T16:45:13,720 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:45629 connecting to ZooKeeper ensemble=127.0.0.1:60963 2024-11-24T16:45:13,755 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:456290x0, quorum=127.0.0.1:60963, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-24T16:45:13,758 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:45629-0x1014ade0b8c0000 connected 2024-11-24T16:45:13,787 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-24T16:45:13,788 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-24T16:45:13,798 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:45629-0x1014ade0b8c0000, quorum=127.0.0.1:60963, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-24T16:45:13,802 INFO [Time-limited test {}] master.HMaster(525): hbase.rootdir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9797cf4c-7cf0-a873-8ba7-f30267b4a2f7/39f4fdd4-f188-1d18-4286-5b0c4dc27cad, hbase.cluster.distributed=false 2024-11-24T16:45:13,830 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:45629-0x1014ade0b8c0000, quorum=127.0.0.1:60963, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-24T16:45:13,836 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=45629 2024-11-24T16:45:13,836 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=45629 2024-11-24T16:45:13,837 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=45629 2024-11-24T16:45:13,837 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=45629 2024-11-24T16:45:13,838 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=45629 2024-11-24T16:45:13,928 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/e335232f7491:0 server-side Connection retries=45 2024-11-24T16:45:13,930 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-24T16:45:13,930 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-24T16:45:13,931 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-24T16:45:13,931 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-24T16:45:13,931 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-24T16:45:13,934 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-24T16:45:13,938 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-24T16:45:13,939 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:45465 2024-11-24T16:45:13,941 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:45465 connecting to ZooKeeper ensemble=127.0.0.1:60963 2024-11-24T16:45:13,942 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-24T16:45:13,943 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-24T16:45:13,947 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:454650x0, quorum=127.0.0.1:60963, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-24T16:45:13,948 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:45465-0x1014ade0b8c0001 connected 2024-11-24T16:45:13,948 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:45465-0x1014ade0b8c0001, quorum=127.0.0.1:60963, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-24T16:45:13,954 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-24T16:45:13,969 INFO [Time-limited test {}] bucket.FileIOEngine(99): Allocating cache 400 MB, on the path:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9797cf4c-7cf0-a873-8ba7-f30267b4a2f7/bucket.cache 2024-11-24T16:45:13,977 INFO [Time-limited test {}] bucket.BucketCache(364): Instantiating BucketCache with acceptableFactor: 0.95, minFactor: 0.85, extraFreeFactor: 0.1, singleFactor: 0.25, multiFactor: 0.5, memoryFactor: 0.25, useStrongRef: false 2024-11-24T16:45:13,979 INFO [Time-limited test {}] bucket.BucketCache(478): Starting BucketCachePersisterThread 2024-11-24T16:45:13,981 INFO [Time-limited test {}] bucket.BucketCachePersister(38): BucketCachePersister started with interval: 100 2024-11-24T16:45:13,983 INFO [Thread-10 {}] bucket.BucketCache(1530): Started retrieving bucket cache from file 2024-11-24T16:45:13,983 WARN [Thread-10 {}] bucket.BucketCache(1533): Persistence file missing! It's ok if it's first run after enabling persistent cache. 2024-11-24T16:45:13,988 INFO [Time-limited test {}] bucket.BucketCache(403): Started bucket cache; ioengine=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9797cf4c-7cf0-a873-8ba7-f30267b4a2f7/bucket.cache, capacity=400 MB, blockSize=64 KB, writerThreadNum=3, writerQLen=64, persistencePath=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9797cf4c-7cf0-a873-8ba7-f30267b4a2f7/bucket.persistence, bucketAllocator=org.apache.hadoop.hbase.io.hfile.bucket.BucketAllocator 2024-11-24T16:45:13,994 INFO [Thread-10 {}] bucket.BucketAllocator(343): Cache totalSize=418148352, buckets=199, bucket capacity=2101248=(4*525312)=(FEWEST_ITEMS_IN_BUCKET*(largest configured bucketcache size)) 2024-11-24T16:45:13,994 INFO [Thread-10 {}] bucket.BucketCache(413): Persistent bucket cache recovery from /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9797cf4c-7cf0-a873-8ba7-f30267b4a2f7/bucket.persistence is complete. 2024-11-24T16:45:13,995 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-24T16:45:13,998 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:45465-0x1014ade0b8c0001, quorum=127.0.0.1:60963, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-24T16:45:14,005 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:45465-0x1014ade0b8c0001, quorum=127.0.0.1:60963, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-24T16:45:14,006 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=45465 2024-11-24T16:45:14,006 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=45465 2024-11-24T16:45:14,008 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=45465 2024-11-24T16:45:14,009 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=45465 2024-11-24T16:45:14,011 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=45465 2024-11-24T16:45:14,030 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/e335232f7491:0 server-side Connection retries=45 2024-11-24T16:45:14,031 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-24T16:45:14,031 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-24T16:45:14,031 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-24T16:45:14,031 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-24T16:45:14,031 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-24T16:45:14,031 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-24T16:45:14,032 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-24T16:45:14,033 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:39717 2024-11-24T16:45:14,035 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:39717 connecting to ZooKeeper ensemble=127.0.0.1:60963 2024-11-24T16:45:14,035 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-24T16:45:14,036 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-24T16:45:14,041 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:397170x0, quorum=127.0.0.1:60963, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-24T16:45:14,042 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:397170x0, quorum=127.0.0.1:60963, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-24T16:45:14,042 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:39717-0x1014ade0b8c0002 connected 2024-11-24T16:45:14,043 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-24T16:45:14,044 INFO [Time-limited test {}] bucket.FileIOEngine(99): Allocating cache 400 MB, on the path:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9797cf4c-7cf0-a873-8ba7-f30267b4a2f7/bucket.cache 2024-11-24T16:45:14,044 INFO [Time-limited test {}] bucket.BucketCache(364): Instantiating BucketCache with acceptableFactor: 0.95, minFactor: 0.85, extraFreeFactor: 0.1, singleFactor: 0.25, multiFactor: 0.5, memoryFactor: 0.25, useStrongRef: false 2024-11-24T16:45:14,045 INFO [Time-limited test {}] bucket.BucketCache(478): Starting BucketCachePersisterThread 2024-11-24T16:45:14,045 INFO [Time-limited test {}] bucket.BucketCachePersister(38): BucketCachePersister started with interval: 100 2024-11-24T16:45:14,046 INFO [Thread-11 {}] bucket.BucketCache(1530): Started retrieving bucket cache from file 2024-11-24T16:45:14,046 WARN [Thread-11 {}] bucket.BucketCache(1533): Persistence file missing! It's ok if it's first run after enabling persistent cache. 2024-11-24T16:45:14,047 INFO [Thread-11 {}] bucket.BucketAllocator(343): Cache totalSize=418148352, buckets=199, bucket capacity=2101248=(4*525312)=(FEWEST_ITEMS_IN_BUCKET*(largest configured bucketcache size)) 2024-11-24T16:45:14,047 INFO [Thread-11 {}] bucket.BucketCache(413): Persistent bucket cache recovery from /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9797cf4c-7cf0-a873-8ba7-f30267b4a2f7/bucket.persistence is complete. 2024-11-24T16:45:14,048 INFO [Time-limited test {}] bucket.BucketCache(403): Started bucket cache; ioengine=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9797cf4c-7cf0-a873-8ba7-f30267b4a2f7/bucket.cache, capacity=400 MB, blockSize=64 KB, writerThreadNum=3, writerQLen=64, persistencePath=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9797cf4c-7cf0-a873-8ba7-f30267b4a2f7/bucket.persistence, bucketAllocator=org.apache.hadoop.hbase.io.hfile.bucket.BucketAllocator 2024-11-24T16:45:14,050 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-24T16:45:14,051 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:39717-0x1014ade0b8c0002, quorum=127.0.0.1:60963, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-24T16:45:14,054 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:39717-0x1014ade0b8c0002, quorum=127.0.0.1:60963, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-24T16:45:14,054 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=39717 2024-11-24T16:45:14,055 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=39717 2024-11-24T16:45:14,060 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=39717 2024-11-24T16:45:14,060 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=39717 2024-11-24T16:45:14,061 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=39717 2024-11-24T16:45:14,079 DEBUG [M:0;e335232f7491:45629 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;e335232f7491:45629 2024-11-24T16:45:14,081 INFO [master/e335232f7491:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/e335232f7491,45629,1732466713116 2024-11-24T16:45:14,089 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45629-0x1014ade0b8c0000, quorum=127.0.0.1:60963, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-24T16:45:14,089 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45465-0x1014ade0b8c0001, quorum=127.0.0.1:60963, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-24T16:45:14,089 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39717-0x1014ade0b8c0002, quorum=127.0.0.1:60963, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-24T16:45:14,091 DEBUG [master/e335232f7491:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:45629-0x1014ade0b8c0000, quorum=127.0.0.1:60963, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/e335232f7491,45629,1732466713116 2024-11-24T16:45:14,117 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39717-0x1014ade0b8c0002, quorum=127.0.0.1:60963, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-24T16:45:14,117 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45465-0x1014ade0b8c0001, quorum=127.0.0.1:60963, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-24T16:45:14,117 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45629-0x1014ade0b8c0000, quorum=127.0.0.1:60963, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T16:45:14,117 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45465-0x1014ade0b8c0001, quorum=127.0.0.1:60963, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T16:45:14,117 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39717-0x1014ade0b8c0002, quorum=127.0.0.1:60963, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T16:45:14,119 DEBUG [master/e335232f7491:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:45629-0x1014ade0b8c0000, quorum=127.0.0.1:60963, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-11-24T16:45:14,120 INFO [master/e335232f7491:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/e335232f7491,45629,1732466713116 from backup master directory 2024-11-24T16:45:14,123 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45629-0x1014ade0b8c0000, quorum=127.0.0.1:60963, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/e335232f7491,45629,1732466713116 2024-11-24T16:45:14,123 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39717-0x1014ade0b8c0002, quorum=127.0.0.1:60963, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-24T16:45:14,123 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45465-0x1014ade0b8c0001, quorum=127.0.0.1:60963, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-24T16:45:14,123 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45629-0x1014ade0b8c0000, quorum=127.0.0.1:60963, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-24T16:45:14,124 WARN [master/e335232f7491:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-24T16:45:14,124 INFO [master/e335232f7491:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=e335232f7491,45629,1732466713116 2024-11-24T16:45:14,127 INFO [master/e335232f7491:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating data MemStoreChunkPool with chunk size 2 MB, max count 396, initial count 0 2024-11-24T16:45:14,130 INFO [master/e335232f7491:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating index MemStoreChunkPool with chunk size 204.80 KB, max count 440, initial count 0 2024-11-24T16:45:14,140 DEBUG [master/e335232f7491:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9797cf4c-7cf0-a873-8ba7-f30267b4a2f7/39f4fdd4-f188-1d18-4286-5b0c4dc27cad/hbase.id] with ID: b0e67acc-6890-4b90-9c2d-c4b4b74b4a71 2024-11-24T16:45:14,140 DEBUG [master/e335232f7491:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9797cf4c-7cf0-a873-8ba7-f30267b4a2f7/39f4fdd4-f188-1d18-4286-5b0c4dc27cad/.tmp/hbase.id 2024-11-24T16:45:14,147 DEBUG [master/e335232f7491:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9797cf4c-7cf0-a873-8ba7-f30267b4a2f7/39f4fdd4-f188-1d18-4286-5b0c4dc27cad/.tmp/hbase.id]:[file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9797cf4c-7cf0-a873-8ba7-f30267b4a2f7/39f4fdd4-f188-1d18-4286-5b0c4dc27cad/hbase.id] 2024-11-24T16:45:14,188 DEBUG [master/e335232f7491:0:becomeActiveMaster {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-24T16:45:14,194 INFO [master/e335232f7491:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem. 2024-11-24T16:45:14,215 INFO [master/e335232f7491:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 18ms. 2024-11-24T16:45:14,220 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45465-0x1014ade0b8c0001, quorum=127.0.0.1:60963, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T16:45:14,220 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45629-0x1014ade0b8c0000, quorum=127.0.0.1:60963, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T16:45:14,220 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39717-0x1014ade0b8c0002, quorum=127.0.0.1:60963, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T16:45:14,248 INFO [master/e335232f7491:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-24T16:45:14,251 INFO [master/e335232f7491:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-11-24T16:45:14,298 DEBUG [master/e335232f7491:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(244): No decryptEncryptedDataEncryptionKey method in DFSClient, should be hadoop version with HDFS-12396 java.lang.NoSuchMethodException: org.apache.hadoop.hdfs.DFSClient.decryptEncryptedDataEncryptionKey(org.apache.hadoop.fs.FileEncryptionInfo) at java.lang.Class.getDeclaredMethod(Class.java:2675) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelperWithoutHDFS12396(FanOutOneBlockAsyncDFSOutputSaslHelper.java:183) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelper(FanOutOneBlockAsyncDFSOutputSaslHelper.java:242) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.(FanOutOneBlockAsyncDFSOutputSaslHelper.java:253) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.lang.Class.forName0(Native Method) ~[?:?] at java.lang.Class.forName(Class.java:375) ~[?:?] at org.apache.hadoop.hbase.wal.AsyncFSWALProvider.load(AsyncFSWALProvider.java:150) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.getProviderClass(WALFactory.java:174) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:262) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:231) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegion.create(MasterRegion.java:400) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegionFactory.create(MasterRegionFactory.java:135) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.finishActiveMasterInitialization(HMaster.java:1003) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.startActiveMasterManager(HMaster.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.lambda$run$0(HMaster.java:613) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.lambda$tracedRunnable$2(TraceUtil.java:155) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T16:45:14,305 INFO [master/e335232f7491:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-24T16:45:14,367 INFO [master/e335232f7491:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9797cf4c-7cf0-a873-8ba7-f30267b4a2f7/39f4fdd4-f188-1d18-4286-5b0c4dc27cad/MasterData/data/master/store 2024-11-24T16:45:14,387 INFO [master/e335232f7491:0:becomeActiveMaster {}] throttle.StoreHotnessProtector(112): StoreHotnessProtector is disabled. Set hbase.region.store.parallel.put.limit > 0 to enable, which may help mitigate load under heavy write pressure. 2024-11-24T16:45:14,390 DEBUG [master/e335232f7491:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T16:45:14,391 DEBUG [master/e335232f7491:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-24T16:45:14,391 INFO [master/e335232f7491:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-24T16:45:14,392 DEBUG [master/e335232f7491:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-24T16:45:14,393 DEBUG [master/e335232f7491:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-24T16:45:14,393 DEBUG [master/e335232f7491:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-24T16:45:14,393 INFO [master/e335232f7491:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-24T16:45:14,394 DEBUG [master/e335232f7491:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1732466714391Disabling compacts and flushes for region at 1732466714391Disabling writes for close at 1732466714393 (+2 ms)Writing region close event to WAL at 1732466714393Closed at 1732466714393 2024-11-24T16:45:14,394 WARN [master/e335232f7491:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9797cf4c-7cf0-a873-8ba7-f30267b4a2f7/39f4fdd4-f188-1d18-4286-5b0c4dc27cad/MasterData/data/master/store/.initializing 2024-11-24T16:45:14,395 DEBUG [master/e335232f7491:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9797cf4c-7cf0-a873-8ba7-f30267b4a2f7/39f4fdd4-f188-1d18-4286-5b0c4dc27cad/MasterData/WALs/e335232f7491,45629,1732466713116 2024-11-24T16:45:14,408 INFO [master/e335232f7491:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-24T16:45:14,421 INFO [master/e335232f7491:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=e335232f7491%2C45629%2C1732466713116, suffix=, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9797cf4c-7cf0-a873-8ba7-f30267b4a2f7/39f4fdd4-f188-1d18-4286-5b0c4dc27cad/MasterData/WALs/e335232f7491,45629,1732466713116, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9797cf4c-7cf0-a873-8ba7-f30267b4a2f7/39f4fdd4-f188-1d18-4286-5b0c4dc27cad/MasterData/oldWALs, maxLogs=10 2024-11-24T16:45:14,459 INFO [master/e335232f7491:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9797cf4c-7cf0-a873-8ba7-f30267b4a2f7/39f4fdd4-f188-1d18-4286-5b0c4dc27cad/MasterData/WALs/e335232f7491,45629,1732466713116/e335232f7491%2C45629%2C1732466713116.1732466714426 2024-11-24T16:45:14,460 DEBUG [master/e335232f7491:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-11-24T16:45:14,460 DEBUG [master/e335232f7491:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-11-24T16:45:14,461 DEBUG [master/e335232f7491:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T16:45:14,464 DEBUG [master/e335232f7491:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-11-24T16:45:14,465 DEBUG [master/e335232f7491:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-11-24T16:45:14,502 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-11-24T16:45:14,537 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-11-24T16:45:14,541 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:45:14,543 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-24T16:45:14,544 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-11-24T16:45:14,549 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-11-24T16:45:14,549 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:45:14,550 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T16:45:14,550 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-11-24T16:45:14,555 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-11-24T16:45:14,555 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:45:14,556 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T16:45:14,556 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-11-24T16:45:14,561 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-11-24T16:45:14,561 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:45:14,561 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T16:45:14,562 DEBUG [master/e335232f7491:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682 2024-11-24T16:45:14,565 DEBUG [master/e335232f7491:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9797cf4c-7cf0-a873-8ba7-f30267b4a2f7/39f4fdd4-f188-1d18-4286-5b0c4dc27cad/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-24T16:45:14,566 DEBUG [master/e335232f7491:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9797cf4c-7cf0-a873-8ba7-f30267b4a2f7/39f4fdd4-f188-1d18-4286-5b0c4dc27cad/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-24T16:45:14,569 DEBUG [master/e335232f7491:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682 2024-11-24T16:45:14,569 DEBUG [master/e335232f7491:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682 2024-11-24T16:45:14,572 DEBUG [master/e335232f7491:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-24T16:45:14,574 DEBUG [master/e335232f7491:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-11-24T16:45:14,583 DEBUG [master/e335232f7491:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9797cf4c-7cf0-a873-8ba7-f30267b4a2f7/39f4fdd4-f188-1d18-4286-5b0c4dc27cad/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-24T16:45:14,584 INFO [master/e335232f7491:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=66518320, jitterRate=-0.00879979133605957}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-24T16:45:14,591 DEBUG [master/e335232f7491:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682: Writing region info on filesystem at 1732466714478Initializing all the Stores at 1732466714478Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732466714479 (+1 ms)Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732466714479Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732466714480 (+1 ms)Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732466714480Cleaning up temporary data from old regions at 1732466714569 (+89 ms)Region opened successfully at 1732466714591 (+22 ms) 2024-11-24T16:45:14,593 INFO [master/e335232f7491:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-11-24T16:45:14,629 DEBUG [master/e335232f7491:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7eb43c34, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=e335232f7491/172.17.0.2:0 2024-11-24T16:45:14,666 INFO [master/e335232f7491:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating... 2024-11-24T16:45:14,678 INFO [master/e335232f7491:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5 2024-11-24T16:45:14,678 INFO [master/e335232f7491:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50 2024-11-24T16:45:14,681 INFO [master/e335232f7491:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-24T16:45:14,682 INFO [master/e335232f7491:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-24T16:45:14,688 INFO [master/e335232f7491:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 5 msec 2024-11-24T16:45:14,688 INFO [master/e335232f7491:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-11-24T16:45:14,715 INFO [master/e335232f7491:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2024-11-24T16:45:14,727 DEBUG [master/e335232f7491:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:45629-0x1014ade0b8c0000, quorum=127.0.0.1:60963, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error) 2024-11-24T16:45:14,730 DEBUG [master/e335232f7491:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false 2024-11-24T16:45:14,734 INFO [master/e335232f7491:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2024-11-24T16:45:14,736 DEBUG [master/e335232f7491:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:45629-0x1014ade0b8c0000, quorum=127.0.0.1:60963, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error) 2024-11-24T16:45:14,738 DEBUG [master/e335232f7491:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false 2024-11-24T16:45:14,741 INFO [master/e335232f7491:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2024-11-24T16:45:14,746 DEBUG [master/e335232f7491:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:45629-0x1014ade0b8c0000, quorum=127.0.0.1:60963, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error) 2024-11-24T16:45:14,748 DEBUG [master/e335232f7491:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false 2024-11-24T16:45:14,750 DEBUG [master/e335232f7491:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:45629-0x1014ade0b8c0000, quorum=127.0.0.1:60963, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error) 2024-11-24T16:45:14,752 DEBUG [master/e335232f7491:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false 2024-11-24T16:45:14,775 DEBUG [master/e335232f7491:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:45629-0x1014ade0b8c0000, quorum=127.0.0.1:60963, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error) 2024-11-24T16:45:14,777 DEBUG [master/e335232f7491:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false 2024-11-24T16:45:14,791 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39717-0x1014ade0b8c0002, quorum=127.0.0.1:60963, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-24T16:45:14,791 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45465-0x1014ade0b8c0001, quorum=127.0.0.1:60963, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-24T16:45:14,791 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45629-0x1014ade0b8c0000, quorum=127.0.0.1:60963, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-24T16:45:14,791 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39717-0x1014ade0b8c0002, quorum=127.0.0.1:60963, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T16:45:14,791 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45465-0x1014ade0b8c0001, quorum=127.0.0.1:60963, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T16:45:14,791 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45629-0x1014ade0b8c0000, quorum=127.0.0.1:60963, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T16:45:14,794 INFO [master/e335232f7491:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=e335232f7491,45629,1732466713116, sessionid=0x1014ade0b8c0000, setting cluster-up flag (Was=false) 2024-11-24T16:45:14,806 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39717-0x1014ade0b8c0002, quorum=127.0.0.1:60963, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T16:45:14,806 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45629-0x1014ade0b8c0000, quorum=127.0.0.1:60963, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T16:45:14,806 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45465-0x1014ade0b8c0001, quorum=127.0.0.1:60963, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T16:45:14,813 DEBUG [master/e335232f7491:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort 2024-11-24T16:45:14,815 DEBUG [master/e335232f7491:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=e335232f7491,45629,1732466713116 2024-11-24T16:45:14,821 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39717-0x1014ade0b8c0002, quorum=127.0.0.1:60963, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T16:45:14,821 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45465-0x1014ade0b8c0001, quorum=127.0.0.1:60963, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T16:45:14,821 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45629-0x1014ade0b8c0000, quorum=127.0.0.1:60963, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T16:45:14,826 DEBUG [master/e335232f7491:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort 2024-11-24T16:45:14,828 DEBUG [master/e335232f7491:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=e335232f7491,45629,1732466713116 2024-11-24T16:45:14,832 INFO [master/e335232f7491:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9797cf4c-7cf0-a873-8ba7-f30267b4a2f7/39f4fdd4-f188-1d18-4286-5b0c4dc27cad/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again 2024-11-24T16:45:14,867 INFO [RS:1;e335232f7491:39717 {}] regionserver.HRegionServer(746): ClusterId : b0e67acc-6890-4b90-9c2d-c4b4b74b4a71 2024-11-24T16:45:14,867 INFO [RS:0;e335232f7491:45465 {}] regionserver.HRegionServer(746): ClusterId : b0e67acc-6890-4b90-9c2d-c4b4b74b4a71 2024-11-24T16:45:14,870 DEBUG [RS:0;e335232f7491:45465 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-24T16:45:14,870 DEBUG [RS:1;e335232f7491:39717 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-24T16:45:14,877 DEBUG [RS:0;e335232f7491:45465 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-24T16:45:14,877 DEBUG [RS:1;e335232f7491:39717 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-24T16:45:14,877 DEBUG [RS:0;e335232f7491:45465 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-24T16:45:14,877 DEBUG [RS:1;e335232f7491:39717 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-24T16:45:14,880 DEBUG [RS:0;e335232f7491:45465 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-24T16:45:14,881 DEBUG [RS:1;e335232f7491:39717 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-24T16:45:14,881 DEBUG [RS:0;e335232f7491:45465 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3752f5c7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=e335232f7491/172.17.0.2:0 2024-11-24T16:45:14,881 DEBUG [RS:1;e335232f7491:39717 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6443907a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=e335232f7491/172.17.0.2:0 2024-11-24T16:45:14,900 DEBUG [RS:0;e335232f7491:45465 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;e335232f7491:45465 2024-11-24T16:45:14,903 DEBUG [RS:1;e335232f7491:39717 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:1;e335232f7491:39717 2024-11-24T16:45:14,904 INFO [RS:0;e335232f7491:45465 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-24T16:45:14,904 INFO [RS:1;e335232f7491:39717 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-24T16:45:14,904 INFO [RS:0;e335232f7491:45465 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-24T16:45:14,904 INFO [RS:1;e335232f7491:39717 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-24T16:45:14,904 DEBUG [RS:0;e335232f7491:45465 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-24T16:45:14,904 DEBUG [RS:1;e335232f7491:39717 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-24T16:45:14,908 INFO [RS:0;e335232f7491:45465 {}] regionserver.HRegionServer(2659): reportForDuty to master=e335232f7491,45629,1732466713116 with port=45465, startcode=1732466713886 2024-11-24T16:45:14,909 INFO [RS:1;e335232f7491:39717 {}] regionserver.HRegionServer(2659): reportForDuty to master=e335232f7491,45629,1732466713116 with port=39717, startcode=1732466714030 2024-11-24T16:45:14,922 DEBUG [RS:0;e335232f7491:45465 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-24T16:45:14,922 DEBUG [RS:1;e335232f7491:39717 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-24T16:45:14,924 DEBUG [master/e335232f7491:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-24T16:45:14,936 INFO [master/e335232f7491:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2 2024-11-24T16:45:14,945 INFO [master/e335232f7491:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2024-11-24T16:45:14,953 DEBUG [master/e335232f7491:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: e335232f7491,45629,1732466713116 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2024-11-24T16:45:14,962 DEBUG [master/e335232f7491:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/e335232f7491:0, corePoolSize=5, maxPoolSize=5 2024-11-24T16:45:14,963 DEBUG [master/e335232f7491:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/e335232f7491:0, corePoolSize=5, maxPoolSize=5 2024-11-24T16:45:14,963 DEBUG [master/e335232f7491:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/e335232f7491:0, corePoolSize=5, maxPoolSize=5 2024-11-24T16:45:14,964 DEBUG [master/e335232f7491:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/e335232f7491:0, corePoolSize=5, maxPoolSize=5 2024-11-24T16:45:14,964 DEBUG [master/e335232f7491:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/e335232f7491:0, corePoolSize=10, maxPoolSize=10 2024-11-24T16:45:14,964 DEBUG [master/e335232f7491:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/e335232f7491:0, corePoolSize=1, maxPoolSize=1 2024-11-24T16:45:14,965 DEBUG [master/e335232f7491:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/e335232f7491:0, corePoolSize=2, maxPoolSize=2 2024-11-24T16:45:14,965 DEBUG [master/e335232f7491:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/e335232f7491:0, corePoolSize=1, maxPoolSize=1 2024-11-24T16:45:14,974 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-24T16:45:14,975 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region 2024-11-24T16:45:14,981 INFO [master/e335232f7491:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732466744981 2024-11-24T16:45:14,983 INFO [master/e335232f7491:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2024-11-24T16:45:14,983 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:45:14,984 INFO [master/e335232f7491:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2024-11-24T16:45:14,984 INFO [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-11-24T16:45:14,987 INFO [master/e335232f7491:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2024-11-24T16:45:14,987 INFO [master/e335232f7491:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2024-11-24T16:45:14,988 INFO [master/e335232f7491:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2024-11-24T16:45:14,988 INFO [master/e335232f7491:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2024-11-24T16:45:14,996 INFO [master/e335232f7491:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-24T16:45:15,001 INFO [master/e335232f7491:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2024-11-24T16:45:15,002 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46985, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=RegionServerStatusService 2024-11-24T16:45:15,002 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40989, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=RegionServerStatusService 2024-11-24T16:45:15,003 INFO [master/e335232f7491:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2024-11-24T16:45:15,003 INFO [master/e335232f7491:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2024-11-24T16:45:15,005 INFO [master/e335232f7491:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2024-11-24T16:45:15,006 INFO [master/e335232f7491:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2024-11-24T16:45:15,008 DEBUG [master/e335232f7491:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/e335232f7491:0:becomeActiveMaster-HFileCleaner.large.0-1732466715007,5,FailOnTimeoutGroup] 2024-11-24T16:45:15,008 DEBUG [master/e335232f7491:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/e335232f7491:0:becomeActiveMaster-HFileCleaner.small.0-1732466715008,5,FailOnTimeoutGroup] 2024-11-24T16:45:15,008 INFO [master/e335232f7491:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-24T16:45:15,010 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=45629 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerStartup(MasterRpcServices.java:667) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16714) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:45:15,010 INFO [master/e335232f7491:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2024-11-24T16:45:15,012 INFO [master/e335232f7491:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2024-11-24T16:45:15,012 INFO [master/e335232f7491:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2024-11-24T16:45:15,017 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=45629 {}] master.ServerManager(363): Checking decommissioned status of RegionServer e335232f7491,45465,1732466713886 2024-11-24T16:45:15,020 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=45629 {}] master.ServerManager(517): Registering regionserver=e335232f7491,45465,1732466713886 2024-11-24T16:45:15,029 INFO [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9797cf4c-7cf0-a873-8ba7-f30267b4a2f7/39f4fdd4-f188-1d18-4286-5b0c4dc27cad/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321 2024-11-24T16:45:15,029 DEBUG [RS:0;e335232f7491:45465 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9797cf4c-7cf0-a873-8ba7-f30267b4a2f7/39f4fdd4-f188-1d18-4286-5b0c4dc27cad 2024-11-24T16:45:15,029 DEBUG [RS:0;e335232f7491:45465 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=file:/ 2024-11-24T16:45:15,029 INFO [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9797cf4c-7cf0-a873-8ba7-f30267b4a2f7/39f4fdd4-f188-1d18-4286-5b0c4dc27cad 2024-11-24T16:45:15,029 DEBUG [RS:0;e335232f7491:45465 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-24T16:45:15,035 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45629-0x1014ade0b8c0000, quorum=127.0.0.1:60963, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-24T16:45:15,036 DEBUG [RS:0;e335232f7491:45465 {}] zookeeper.ZKUtil(111): regionserver:45465-0x1014ade0b8c0001, quorum=127.0.0.1:60963, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/e335232f7491,45465,1732466713886 2024-11-24T16:45:15,036 WARN [RS:0;e335232f7491:45465 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-24T16:45:15,036 INFO [RS:0;e335232f7491:45465 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-24T16:45:15,037 DEBUG [RS:0;e335232f7491:45465 {}] regionserver.HRegionServer(1793): logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9797cf4c-7cf0-a873-8ba7-f30267b4a2f7/39f4fdd4-f188-1d18-4286-5b0c4dc27cad/WALs/e335232f7491,45465,1732466713886 2024-11-24T16:45:15,038 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T16:45:15,040 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [e335232f7491,45465,1732466713886] 2024-11-24T16:45:15,044 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family info of region 1588230740 2024-11-24T16:45:15,049 DEBUG [RS:1;e335232f7491:39717 {}] regionserver.HRegionServer(2683): Master is not running yet 2024-11-24T16:45:15,049 WARN [RS:1;e335232f7491:39717 {}] regionserver.HRegionServer(841): reportForDuty failed; sleeping 100 ms and then retrying. 2024-11-24T16:45:15,053 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-24T16:45:15,053 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:45:15,053 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-24T16:45:15,054 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family ns of region 1588230740 2024-11-24T16:45:15,061 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-24T16:45:15,062 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:45:15,062 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-24T16:45:15,062 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family rep_barrier of region 1588230740 2024-11-24T16:45:15,067 INFO [RS:0;e335232f7491:45465 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-24T16:45:15,069 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-24T16:45:15,070 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:45:15,070 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-24T16:45:15,070 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family table of region 1588230740 2024-11-24T16:45:15,076 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-24T16:45:15,076 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:45:15,077 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-24T16:45:15,077 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-24T16:45:15,078 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9797cf4c-7cf0-a873-8ba7-f30267b4a2f7/39f4fdd4-f188-1d18-4286-5b0c4dc27cad/data/hbase/meta/1588230740 2024-11-24T16:45:15,078 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9797cf4c-7cf0-a873-8ba7-f30267b4a2f7/39f4fdd4-f188-1d18-4286-5b0c4dc27cad/data/hbase/meta/1588230740 2024-11-24T16:45:15,079 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-24T16:45:15,079 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-24T16:45:15,079 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-24T16:45:15,080 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-24T16:45:15,086 INFO [RS:0;e335232f7491:45465 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-24T16:45:15,086 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9797cf4c-7cf0-a873-8ba7-f30267b4a2f7/39f4fdd4-f188-1d18-4286-5b0c4dc27cad/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-24T16:45:15,087 INFO [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=64467093, jitterRate=-0.03936545550823212}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-24T16:45:15,091 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740: Writing region info on filesystem at 1732466715039Initializing all the Stores at 1732466715039Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732466715039Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732466715040 (+1 ms)Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732466715040Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732466715040Cleaning up temporary data from old regions at 1732466715079 (+39 ms)Region opened successfully at 1732466715091 (+12 ms) 2024-11-24T16:45:15,091 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-24T16:45:15,091 INFO [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-24T16:45:15,091 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-24T16:45:15,091 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-24T16:45:15,091 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-24T16:45:15,092 INFO [RS:0;e335232f7491:45465 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-24T16:45:15,093 INFO [RS:0;e335232f7491:45465 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-24T16:45:15,093 INFO [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-24T16:45:15,093 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1732466715091Disabling compacts and flushes for region at 1732466715091Disabling writes for close at 1732466715091Writing region close event to WAL at 1732466715093 (+2 ms)Closed at 1732466715093 2024-11-24T16:45:15,094 INFO [RS:0;e335232f7491:45465 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-24T16:45:15,096 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-24T16:45:15,096 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta 2024-11-24T16:45:15,101 INFO [RS:0;e335232f7491:45465 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-24T16:45:15,102 INFO [RS:0;e335232f7491:45465 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-24T16:45:15,103 DEBUG [RS:0;e335232f7491:45465 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/e335232f7491:0, corePoolSize=1, maxPoolSize=1 2024-11-24T16:45:15,103 DEBUG [RS:0;e335232f7491:45465 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/e335232f7491:0, corePoolSize=1, maxPoolSize=1 2024-11-24T16:45:15,103 DEBUG [RS:0;e335232f7491:45465 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/e335232f7491:0, corePoolSize=1, maxPoolSize=1 2024-11-24T16:45:15,104 DEBUG [RS:0;e335232f7491:45465 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/e335232f7491:0, corePoolSize=1, maxPoolSize=1 2024-11-24T16:45:15,104 DEBUG [RS:0;e335232f7491:45465 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/e335232f7491:0, corePoolSize=1, maxPoolSize=1 2024-11-24T16:45:15,104 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-11-24T16:45:15,104 DEBUG [RS:0;e335232f7491:45465 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/e335232f7491:0, corePoolSize=2, maxPoolSize=2 2024-11-24T16:45:15,104 DEBUG [RS:0;e335232f7491:45465 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/e335232f7491:0, corePoolSize=1, maxPoolSize=1 2024-11-24T16:45:15,104 DEBUG [RS:0;e335232f7491:45465 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/e335232f7491:0, corePoolSize=1, maxPoolSize=1 2024-11-24T16:45:15,105 DEBUG [RS:0;e335232f7491:45465 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/e335232f7491:0, corePoolSize=1, maxPoolSize=1 2024-11-24T16:45:15,105 DEBUG [RS:0;e335232f7491:45465 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/e335232f7491:0, corePoolSize=1, maxPoolSize=1 2024-11-24T16:45:15,105 DEBUG [RS:0;e335232f7491:45465 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/e335232f7491:0, corePoolSize=1, maxPoolSize=1 2024-11-24T16:45:15,105 DEBUG [RS:0;e335232f7491:45465 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/e335232f7491:0, corePoolSize=1, maxPoolSize=1 2024-11-24T16:45:15,106 DEBUG [RS:0;e335232f7491:45465 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/e335232f7491:0, corePoolSize=3, maxPoolSize=3 2024-11-24T16:45:15,106 DEBUG [RS:0;e335232f7491:45465 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0, corePoolSize=3, maxPoolSize=3 2024-11-24T16:45:15,108 INFO [RS:0;e335232f7491:45465 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-24T16:45:15,108 INFO [RS:0;e335232f7491:45465 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-24T16:45:15,108 INFO [RS:0;e335232f7491:45465 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-24T16:45:15,108 INFO [RS:0;e335232f7491:45465 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-24T16:45:15,109 INFO [RS:0;e335232f7491:45465 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-24T16:45:15,109 INFO [RS:0;e335232f7491:45465 {}] hbase.ChoreService(168): Chore ScheduledChore name=e335232f7491,45465,1732466713886-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-24T16:45:15,115 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-24T16:45:15,118 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2024-11-24T16:45:15,132 INFO [RS:0;e335232f7491:45465 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-24T16:45:15,134 INFO [RS:0;e335232f7491:45465 {}] hbase.ChoreService(168): Chore ScheduledChore name=e335232f7491,45465,1732466713886-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-24T16:45:15,134 INFO [RS:0;e335232f7491:45465 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-24T16:45:15,134 INFO [RS:0;e335232f7491:45465 {}] regionserver.Replication(171): e335232f7491,45465,1732466713886 started 2024-11-24T16:45:15,150 INFO [RS:1;e335232f7491:39717 {}] regionserver.HRegionServer(2659): reportForDuty to master=e335232f7491,45629,1732466713116 with port=39717, startcode=1732466714030 2024-11-24T16:45:15,152 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=45629 {}] master.ServerManager(363): Checking decommissioned status of RegionServer e335232f7491,39717,1732466714030 2024-11-24T16:45:15,152 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=45629 {}] master.ServerManager(517): Registering regionserver=e335232f7491,39717,1732466714030 2024-11-24T16:45:15,154 INFO [RS:0;e335232f7491:45465 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-24T16:45:15,154 INFO [RS:0;e335232f7491:45465 {}] regionserver.HRegionServer(1482): Serving as e335232f7491,45465,1732466713886, RpcServer on e335232f7491/172.17.0.2:45465, sessionid=0x1014ade0b8c0001 2024-11-24T16:45:15,155 DEBUG [RS:0;e335232f7491:45465 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-24T16:45:15,155 DEBUG [RS:1;e335232f7491:39717 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9797cf4c-7cf0-a873-8ba7-f30267b4a2f7/39f4fdd4-f188-1d18-4286-5b0c4dc27cad 2024-11-24T16:45:15,155 DEBUG [RS:0;e335232f7491:45465 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager e335232f7491,45465,1732466713886 2024-11-24T16:45:15,155 DEBUG [RS:1;e335232f7491:39717 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=file:/ 2024-11-24T16:45:15,155 DEBUG [RS:1;e335232f7491:39717 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-24T16:45:15,155 DEBUG [RS:0;e335232f7491:45465 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'e335232f7491,45465,1732466713886' 2024-11-24T16:45:15,155 DEBUG [RS:0;e335232f7491:45465 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-24T16:45:15,156 DEBUG [RS:0;e335232f7491:45465 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-24T16:45:15,164 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45629-0x1014ade0b8c0000, quorum=127.0.0.1:60963, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-24T16:45:15,164 DEBUG [RS:0;e335232f7491:45465 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-24T16:45:15,164 DEBUG [RS:0;e335232f7491:45465 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-24T16:45:15,164 DEBUG [RS:0;e335232f7491:45465 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager e335232f7491,45465,1732466713886 2024-11-24T16:45:15,164 DEBUG [RS:0;e335232f7491:45465 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'e335232f7491,45465,1732466713886' 2024-11-24T16:45:15,165 DEBUG [RS:0;e335232f7491:45465 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-24T16:45:15,165 DEBUG [RS:1;e335232f7491:39717 {}] zookeeper.ZKUtil(111): regionserver:39717-0x1014ade0b8c0002, quorum=127.0.0.1:60963, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/e335232f7491,39717,1732466714030 2024-11-24T16:45:15,165 WARN [RS:1;e335232f7491:39717 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-24T16:45:15,165 INFO [RS:1;e335232f7491:39717 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-24T16:45:15,165 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [e335232f7491,39717,1732466714030] 2024-11-24T16:45:15,165 DEBUG [RS:0;e335232f7491:45465 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-24T16:45:15,165 DEBUG [RS:1;e335232f7491:39717 {}] regionserver.HRegionServer(1793): logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9797cf4c-7cf0-a873-8ba7-f30267b4a2f7/39f4fdd4-f188-1d18-4286-5b0c4dc27cad/WALs/e335232f7491,39717,1732466714030 2024-11-24T16:45:15,166 DEBUG [RS:0;e335232f7491:45465 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-24T16:45:15,166 INFO [RS:0;e335232f7491:45465 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-24T16:45:15,166 INFO [RS:0;e335232f7491:45465 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-24T16:45:15,171 INFO [RS:1;e335232f7491:39717 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-24T16:45:15,175 INFO [RS:1;e335232f7491:39717 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-24T16:45:15,175 INFO [RS:1;e335232f7491:39717 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-24T16:45:15,175 INFO [RS:1;e335232f7491:39717 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-24T16:45:15,176 INFO [RS:1;e335232f7491:39717 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-24T16:45:15,176 INFO [RS:1;e335232f7491:39717 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-24T16:45:15,177 INFO [RS:1;e335232f7491:39717 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-24T16:45:15,177 DEBUG [RS:1;e335232f7491:39717 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/e335232f7491:0, corePoolSize=1, maxPoolSize=1 2024-11-24T16:45:15,177 DEBUG [RS:1;e335232f7491:39717 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/e335232f7491:0, corePoolSize=1, maxPoolSize=1 2024-11-24T16:45:15,177 DEBUG [RS:1;e335232f7491:39717 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/e335232f7491:0, corePoolSize=1, maxPoolSize=1 2024-11-24T16:45:15,177 DEBUG [RS:1;e335232f7491:39717 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/e335232f7491:0, corePoolSize=1, maxPoolSize=1 2024-11-24T16:45:15,177 DEBUG [RS:1;e335232f7491:39717 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/e335232f7491:0, corePoolSize=1, maxPoolSize=1 2024-11-24T16:45:15,177 DEBUG [RS:1;e335232f7491:39717 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/e335232f7491:0, corePoolSize=2, maxPoolSize=2 2024-11-24T16:45:15,178 DEBUG [RS:1;e335232f7491:39717 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/e335232f7491:0, corePoolSize=1, maxPoolSize=1 2024-11-24T16:45:15,178 DEBUG [RS:1;e335232f7491:39717 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/e335232f7491:0, corePoolSize=1, maxPoolSize=1 2024-11-24T16:45:15,178 DEBUG [RS:1;e335232f7491:39717 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/e335232f7491:0, corePoolSize=1, maxPoolSize=1 2024-11-24T16:45:15,178 DEBUG [RS:1;e335232f7491:39717 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/e335232f7491:0, corePoolSize=1, maxPoolSize=1 2024-11-24T16:45:15,178 DEBUG [RS:1;e335232f7491:39717 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/e335232f7491:0, corePoolSize=1, maxPoolSize=1 2024-11-24T16:45:15,178 DEBUG [RS:1;e335232f7491:39717 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/e335232f7491:0, corePoolSize=1, maxPoolSize=1 2024-11-24T16:45:15,178 DEBUG [RS:1;e335232f7491:39717 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/e335232f7491:0, corePoolSize=3, maxPoolSize=3 2024-11-24T16:45:15,178 DEBUG [RS:1;e335232f7491:39717 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0, corePoolSize=3, maxPoolSize=3 2024-11-24T16:45:15,181 INFO [RS:1;e335232f7491:39717 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-24T16:45:15,181 INFO [RS:1;e335232f7491:39717 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-24T16:45:15,181 INFO [RS:1;e335232f7491:39717 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-24T16:45:15,181 INFO [RS:1;e335232f7491:39717 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-24T16:45:15,181 INFO [RS:1;e335232f7491:39717 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-24T16:45:15,181 INFO [RS:1;e335232f7491:39717 {}] hbase.ChoreService(168): Chore ScheduledChore name=e335232f7491,39717,1732466714030-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-24T16:45:15,198 INFO [RS:1;e335232f7491:39717 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-24T16:45:15,198 INFO [RS:1;e335232f7491:39717 {}] hbase.ChoreService(168): Chore ScheduledChore name=e335232f7491,39717,1732466714030-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-24T16:45:15,198 INFO [RS:1;e335232f7491:39717 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-24T16:45:15,198 INFO [RS:1;e335232f7491:39717 {}] regionserver.Replication(171): e335232f7491,39717,1732466714030 started 2024-11-24T16:45:15,214 INFO [RS:1;e335232f7491:39717 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-24T16:45:15,215 INFO [RS:1;e335232f7491:39717 {}] regionserver.HRegionServer(1482): Serving as e335232f7491,39717,1732466714030, RpcServer on e335232f7491/172.17.0.2:39717, sessionid=0x1014ade0b8c0002 2024-11-24T16:45:15,215 DEBUG [RS:1;e335232f7491:39717 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-24T16:45:15,215 DEBUG [RS:1;e335232f7491:39717 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager e335232f7491,39717,1732466714030 2024-11-24T16:45:15,215 DEBUG [RS:1;e335232f7491:39717 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'e335232f7491,39717,1732466714030' 2024-11-24T16:45:15,215 DEBUG [RS:1;e335232f7491:39717 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-24T16:45:15,216 DEBUG [RS:1;e335232f7491:39717 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-24T16:45:15,217 DEBUG [RS:1;e335232f7491:39717 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-24T16:45:15,218 DEBUG [RS:1;e335232f7491:39717 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-24T16:45:15,218 DEBUG [RS:1;e335232f7491:39717 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager e335232f7491,39717,1732466714030 2024-11-24T16:45:15,218 DEBUG [RS:1;e335232f7491:39717 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'e335232f7491,39717,1732466714030' 2024-11-24T16:45:15,218 DEBUG [RS:1;e335232f7491:39717 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-24T16:45:15,219 DEBUG [RS:1;e335232f7491:39717 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-24T16:45:15,220 DEBUG [RS:1;e335232f7491:39717 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-24T16:45:15,220 INFO [RS:1;e335232f7491:39717 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-24T16:45:15,220 INFO [RS:1;e335232f7491:39717 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-24T16:45:15,269 WARN [e335232f7491:45629 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-24T16:45:15,272 INFO [RS:0;e335232f7491:45465 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-24T16:45:15,273 INFO [RS:0;e335232f7491:45465 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=e335232f7491%2C45465%2C1732466713886, suffix=, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9797cf4c-7cf0-a873-8ba7-f30267b4a2f7/39f4fdd4-f188-1d18-4286-5b0c4dc27cad/WALs/e335232f7491,45465,1732466713886, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9797cf4c-7cf0-a873-8ba7-f30267b4a2f7/39f4fdd4-f188-1d18-4286-5b0c4dc27cad/oldWALs, maxLogs=55 2024-11-24T16:45:15,294 INFO [RS:0;e335232f7491:45465 {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9797cf4c-7cf0-a873-8ba7-f30267b4a2f7/39f4fdd4-f188-1d18-4286-5b0c4dc27cad/WALs/e335232f7491,45465,1732466713886/e335232f7491%2C45465%2C1732466713886.1732466715275 2024-11-24T16:45:15,294 DEBUG [RS:0;e335232f7491:45465 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-11-24T16:45:15,321 INFO [RS:1;e335232f7491:39717 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-24T16:45:15,322 INFO [RS:1;e335232f7491:39717 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=e335232f7491%2C39717%2C1732466714030, suffix=, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9797cf4c-7cf0-a873-8ba7-f30267b4a2f7/39f4fdd4-f188-1d18-4286-5b0c4dc27cad/WALs/e335232f7491,39717,1732466714030, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9797cf4c-7cf0-a873-8ba7-f30267b4a2f7/39f4fdd4-f188-1d18-4286-5b0c4dc27cad/oldWALs, maxLogs=55 2024-11-24T16:45:15,342 INFO [RS:1;e335232f7491:39717 {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9797cf4c-7cf0-a873-8ba7-f30267b4a2f7/39f4fdd4-f188-1d18-4286-5b0c4dc27cad/WALs/e335232f7491,39717,1732466714030/e335232f7491%2C39717%2C1732466714030.1732466715324 2024-11-24T16:45:15,343 DEBUG [RS:1;e335232f7491:39717 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-11-24T16:45:15,522 DEBUG [e335232f7491:45629 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=2, allServersCount=2 2024-11-24T16:45:15,533 DEBUG [e335232f7491:45629 {}] balancer.BalancerClusterState(204): Hosts are {e335232f7491=0} racks are {/default-rack=0} 2024-11-24T16:45:15,537 DEBUG [e335232f7491:45629 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-11-24T16:45:15,537 DEBUG [e335232f7491:45629 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-11-24T16:45:15,537 DEBUG [e335232f7491:45629 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-11-24T16:45:15,537 DEBUG [e335232f7491:45629 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-11-24T16:45:15,537 INFO [e335232f7491:45629 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-11-24T16:45:15,537 INFO [e335232f7491:45629 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-11-24T16:45:15,538 DEBUG [e335232f7491:45629 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-11-24T16:45:15,545 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=e335232f7491,45465,1732466713886 2024-11-24T16:45:15,552 INFO [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as e335232f7491,45465,1732466713886, state=OPENING 2024-11-24T16:45:15,558 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2024-11-24T16:45:15,560 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45465-0x1014ade0b8c0001, quorum=127.0.0.1:60963, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T16:45:15,560 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39717-0x1014ade0b8c0002, quorum=127.0.0.1:60963, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T16:45:15,560 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45629-0x1014ade0b8c0000, quorum=127.0.0.1:60963, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T16:45:15,561 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-24T16:45:15,561 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-24T16:45:15,561 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-24T16:45:15,563 DEBUG [PEWorker-3 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-24T16:45:15,565 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=e335232f7491,45465,1732466713886}] 2024-11-24T16:45:15,737 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-24T16:45:15,740 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49375, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-24T16:45:15,748 INFO [RS_OPEN_META-regionserver/e335232f7491:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-11-24T16:45:15,749 INFO [RS_OPEN_META-regionserver/e335232f7491:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-24T16:45:15,750 INFO [RS_OPEN_META-regionserver/e335232f7491:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-11-24T16:45:15,751 INFO [RS_OPEN_META-regionserver/e335232f7491:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=e335232f7491%2C45465%2C1732466713886.meta, suffix=.meta, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9797cf4c-7cf0-a873-8ba7-f30267b4a2f7/39f4fdd4-f188-1d18-4286-5b0c4dc27cad/WALs/e335232f7491,45465,1732466713886, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9797cf4c-7cf0-a873-8ba7-f30267b4a2f7/39f4fdd4-f188-1d18-4286-5b0c4dc27cad/oldWALs, maxLogs=55 2024-11-24T16:45:15,775 INFO [RS_OPEN_META-regionserver/e335232f7491:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9797cf4c-7cf0-a873-8ba7-f30267b4a2f7/39f4fdd4-f188-1d18-4286-5b0c4dc27cad/WALs/e335232f7491,45465,1732466713886/e335232f7491%2C45465%2C1732466713886.meta.1732466715753.meta 2024-11-24T16:45:15,775 DEBUG [RS_OPEN_META-regionserver/e335232f7491:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-11-24T16:45:15,776 DEBUG [RS_OPEN_META-regionserver/e335232f7491:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-11-24T16:45:15,778 DEBUG [RS_OPEN_META-regionserver/e335232f7491:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-11-24T16:45:15,780 DEBUG [RS_OPEN_META-regionserver/e335232f7491:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-11-24T16:45:15,785 INFO [RS_OPEN_META-regionserver/e335232f7491:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-11-24T16:45:15,790 DEBUG [RS_OPEN_META-regionserver/e335232f7491:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-11-24T16:45:15,790 DEBUG [RS_OPEN_META-regionserver/e335232f7491:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T16:45:15,791 DEBUG [RS_OPEN_META-regionserver/e335232f7491:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-11-24T16:45:15,791 DEBUG [RS_OPEN_META-regionserver/e335232f7491:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-11-24T16:45:15,793 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family info of region 1588230740 2024-11-24T16:45:15,793 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-24T16:45:15,794 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:45:15,794 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-24T16:45:15,794 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family ns of region 1588230740 2024-11-24T16:45:15,795 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-24T16:45:15,795 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:45:15,796 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-24T16:45:15,796 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family rep_barrier of region 1588230740 2024-11-24T16:45:15,797 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-24T16:45:15,797 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:45:15,797 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-24T16:45:15,797 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family table of region 1588230740 2024-11-24T16:45:15,798 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-24T16:45:15,799 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:45:15,799 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-24T16:45:15,799 DEBUG [RS_OPEN_META-regionserver/e335232f7491:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-24T16:45:15,801 DEBUG [RS_OPEN_META-regionserver/e335232f7491:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9797cf4c-7cf0-a873-8ba7-f30267b4a2f7/39f4fdd4-f188-1d18-4286-5b0c4dc27cad/data/hbase/meta/1588230740 2024-11-24T16:45:15,801 DEBUG [RS_OPEN_META-regionserver/e335232f7491:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9797cf4c-7cf0-a873-8ba7-f30267b4a2f7/39f4fdd4-f188-1d18-4286-5b0c4dc27cad/data/hbase/meta/1588230740 2024-11-24T16:45:15,802 DEBUG [RS_OPEN_META-regionserver/e335232f7491:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-24T16:45:15,802 DEBUG [RS_OPEN_META-regionserver/e335232f7491:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-24T16:45:15,802 DEBUG [RS_OPEN_META-regionserver/e335232f7491:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-24T16:45:15,804 DEBUG [RS_OPEN_META-regionserver/e335232f7491:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-24T16:45:15,805 INFO [RS_OPEN_META-regionserver/e335232f7491:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=63378618, jitterRate=-0.055584996938705444}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-24T16:45:15,805 DEBUG [RS_OPEN_META-regionserver/e335232f7491:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-11-24T16:45:15,807 DEBUG [RS_OPEN_META-regionserver/e335232f7491:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1732466715792Writing region info on filesystem at 1732466715792Initializing all the Stores at 1732466715792Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732466715792Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732466715792Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732466715792Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732466715792Cleaning up temporary data from old regions at 1732466715802 (+10 ms)Running coprocessor post-open hooks at 1732466715805 (+3 ms)Region opened successfully at 1732466715807 (+2 ms) 2024-11-24T16:45:15,814 INFO [RS_OPEN_META-regionserver/e335232f7491:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1732466715728 2024-11-24T16:45:15,827 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=e335232f7491,45465,1732466713886 2024-11-24T16:45:15,828 DEBUG [RS_OPEN_META-regionserver/e335232f7491:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-11-24T16:45:15,828 INFO [RS_OPEN_META-regionserver/e335232f7491:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-11-24T16:45:15,828 INFO [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as e335232f7491,45465,1732466713886, state=OPEN 2024-11-24T16:45:15,831 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39717-0x1014ade0b8c0002, quorum=127.0.0.1:60963, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-24T16:45:15,831 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45465-0x1014ade0b8c0001, quorum=127.0.0.1:60963, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-24T16:45:15,831 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45629-0x1014ade0b8c0000, quorum=127.0.0.1:60963, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-24T16:45:15,831 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-24T16:45:15,831 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-24T16:45:15,831 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-24T16:45:15,832 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=e335232f7491,45465,1732466713886 2024-11-24T16:45:15,835 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2 2024-11-24T16:45:15,835 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=e335232f7491,45465,1732466713886 in 268 msec 2024-11-24T16:45:15,839 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1 2024-11-24T16:45:15,839 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 733 msec 2024-11-24T16:45:15,840 DEBUG [PEWorker-2 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-24T16:45:15,840 INFO [PEWorker-2 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces 2024-11-24T16:45:15,862 DEBUG [PEWorker-2 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-24T16:45:15,863 DEBUG [PEWorker-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=e335232f7491,45465,1732466713886, seqNum=-1] 2024-11-24T16:45:15,883 DEBUG [PEWorker-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-24T16:45:15,886 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50283, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-24T16:45:15,901 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 1.0330 sec 2024-11-24T16:45:15,901 INFO [master/e335232f7491:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1732466715901, completionTime=-1 2024-11-24T16:45:15,904 INFO [master/e335232f7491:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=2; waited=0ms, expected min=2 server(s), max=2 server(s), master is running 2024-11-24T16:45:15,904 DEBUG [master/e335232f7491:0:becomeActiveMaster {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-24T16:45:15,929 INFO [master/e335232f7491:0:becomeActiveMaster {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-11-24T16:45:15,929 INFO [master/e335232f7491:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732466775929 2024-11-24T16:45:15,930 INFO [master/e335232f7491:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732466835929 2024-11-24T16:45:15,930 INFO [master/e335232f7491:0:becomeActiveMaster {}] assignment.AssignmentManager(1783): Joined the cluster in 25 msec 2024-11-24T16:45:15,933 INFO [master/e335232f7491:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=e335232f7491,45629,1732466713116-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-24T16:45:15,934 INFO [master/e335232f7491:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=e335232f7491,45629,1732466713116-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-24T16:45:15,934 INFO [master/e335232f7491:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=e335232f7491,45629,1732466713116-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-24T16:45:15,936 INFO [master/e335232f7491:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-e335232f7491:45629, period=300000, unit=MILLISECONDS is enabled. 2024-11-24T16:45:15,936 INFO [master/e335232f7491:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2024-11-24T16:45:15,937 INFO [master/e335232f7491:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled. 2024-11-24T16:45:15,942 DEBUG [master/e335232f7491:0.Chore.1 {}] janitor.CatalogJanitor(180): 2024-11-24T16:45:15,963 INFO [master/e335232f7491:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 1.838sec 2024-11-24T16:45:15,964 INFO [master/e335232f7491:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2024-11-24T16:45:15,965 INFO [master/e335232f7491:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2024-11-24T16:45:15,966 INFO [master/e335232f7491:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2024-11-24T16:45:15,967 INFO [master/e335232f7491:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2024-11-24T16:45:15,967 INFO [master/e335232f7491:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2024-11-24T16:45:15,968 INFO [master/e335232f7491:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=e335232f7491,45629,1732466713116-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-24T16:45:15,968 INFO [master/e335232f7491:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=e335232f7491,45629,1732466713116-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2024-11-24T16:45:15,971 DEBUG [master/e335232f7491:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds 2024-11-24T16:45:15,972 INFO [master/e335232f7491:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2024-11-24T16:45:15,973 INFO [master/e335232f7491:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=e335232f7491,45629,1732466713116-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-24T16:45:15,977 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@218aa893, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T16:45:15,979 DEBUG [Time-limited test {}] nio.NioEventLoop(110): -Dio.netty.noKeySetOptimization: false 2024-11-24T16:45:15,979 DEBUG [Time-limited test {}] nio.NioEventLoop(111): -Dio.netty.selectorAutoRebuildThreshold: 512 2024-11-24T16:45:15,983 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request e335232f7491,45629,-1 for getting cluster id 2024-11-24T16:45:15,985 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-24T16:45:15,993 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'b0e67acc-6890-4b90-9c2d-c4b4b74b4a71' 2024-11-24T16:45:15,995 DEBUG [RPCClient-NioEventLoopGroup-5-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-24T16:45:15,995 DEBUG [RPCClient-NioEventLoopGroup-5-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "b0e67acc-6890-4b90-9c2d-c4b4b74b4a71" 2024-11-24T16:45:15,995 DEBUG [RPCClient-NioEventLoopGroup-5-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3a50572c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T16:45:15,995 DEBUG [RPCClient-NioEventLoopGroup-5-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [e335232f7491,45629,-1] 2024-11-24T16:45:15,998 DEBUG [RPCClient-NioEventLoopGroup-5-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-24T16:45:15,999 DEBUG [RPCClient-NioEventLoopGroup-5-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T16:45:16,000 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:55104, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-24T16:45:16,003 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@41fe5747, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T16:45:16,003 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-24T16:45:16,010 DEBUG [RPCClient-NioEventLoopGroup-5-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=e335232f7491,45465,1732466713886, seqNum=-1] 2024-11-24T16:45:16,010 DEBUG [RPCClient-NioEventLoopGroup-5-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-24T16:45:16,013 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48466, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-24T16:45:16,044 INFO [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=e335232f7491,45629,1732466713116 2024-11-24T16:45:16,049 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-24T16:45:16,055 DEBUG [RPCClient-NioEventLoopGroup-5-2 {}] client.AsyncConnectionImpl(321): The fetched master address is e335232f7491,45629,1732466713116 2024-11-24T16:45:16,058 DEBUG [RPCClient-NioEventLoopGroup-5-2 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@4c56c949 2024-11-24T16:45:16,060 DEBUG [RPCClient-NioEventLoopGroup-5-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-24T16:45:16,066 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:55120, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-24T16:45:16,072 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45629 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'testBlockEvictionOnRegionMove1732466716044', {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-24T16:45:16,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45629 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=testBlockEvictionOnRegionMove1732466716044 2024-11-24T16:45:16,080 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=testBlockEvictionOnRegionMove1732466716044 execute state=CREATE_TABLE_PRE_OPERATION 2024-11-24T16:45:16,082 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45629 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "testBlockEvictionOnRegionMove1732466716044" procId is: 4 2024-11-24T16:45:16,082 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:45:16,084 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=testBlockEvictionOnRegionMove1732466716044 execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-24T16:45:16,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45629 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-24T16:45:16,097 INFO [RegionOpenAndInit-testBlockEvictionOnRegionMove1732466716044-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 8d688d1c7c98ba065aee654c8fa6e77a, NAME => 'testBlockEvictionOnRegionMove1732466716044,,1732466716067.8d688d1c7c98ba065aee654c8fa6e77a.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testBlockEvictionOnRegionMove1732466716044', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9797cf4c-7cf0-a873-8ba7-f30267b4a2f7/39f4fdd4-f188-1d18-4286-5b0c4dc27cad 2024-11-24T16:45:16,104 DEBUG [RegionOpenAndInit-testBlockEvictionOnRegionMove1732466716044-pool-0 {}] regionserver.HRegion(898): Instantiated testBlockEvictionOnRegionMove1732466716044,,1732466716067.8d688d1c7c98ba065aee654c8fa6e77a.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T16:45:16,104 DEBUG [RegionOpenAndInit-testBlockEvictionOnRegionMove1732466716044-pool-0 {}] regionserver.HRegion(1722): Closing 8d688d1c7c98ba065aee654c8fa6e77a, disabling compactions & flushes 2024-11-24T16:45:16,104 INFO [RegionOpenAndInit-testBlockEvictionOnRegionMove1732466716044-pool-0 {}] regionserver.HRegion(1755): Closing region testBlockEvictionOnRegionMove1732466716044,,1732466716067.8d688d1c7c98ba065aee654c8fa6e77a. 2024-11-24T16:45:16,104 DEBUG [RegionOpenAndInit-testBlockEvictionOnRegionMove1732466716044-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on testBlockEvictionOnRegionMove1732466716044,,1732466716067.8d688d1c7c98ba065aee654c8fa6e77a. 2024-11-24T16:45:16,104 DEBUG [RegionOpenAndInit-testBlockEvictionOnRegionMove1732466716044-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on testBlockEvictionOnRegionMove1732466716044,,1732466716067.8d688d1c7c98ba065aee654c8fa6e77a. after waiting 0 ms 2024-11-24T16:45:16,104 DEBUG [RegionOpenAndInit-testBlockEvictionOnRegionMove1732466716044-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region testBlockEvictionOnRegionMove1732466716044,,1732466716067.8d688d1c7c98ba065aee654c8fa6e77a. 2024-11-24T16:45:16,104 INFO [RegionOpenAndInit-testBlockEvictionOnRegionMove1732466716044-pool-0 {}] regionserver.HRegion(1973): Closed testBlockEvictionOnRegionMove1732466716044,,1732466716067.8d688d1c7c98ba065aee654c8fa6e77a. 2024-11-24T16:45:16,104 DEBUG [RegionOpenAndInit-testBlockEvictionOnRegionMove1732466716044-pool-0 {}] regionserver.HRegion(1676): Region close journal for 8d688d1c7c98ba065aee654c8fa6e77a: Waiting for close lock at 1732466716104Disabling compacts and flushes for region at 1732466716104Disabling writes for close at 1732466716104Writing region close event to WAL at 1732466716104Closed at 1732466716104 2024-11-24T16:45:16,106 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=testBlockEvictionOnRegionMove1732466716044 execute state=CREATE_TABLE_ADD_TO_META 2024-11-24T16:45:16,110 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testBlockEvictionOnRegionMove1732466716044,,1732466716067.8d688d1c7c98ba065aee654c8fa6e77a.","families":{"info":[{"qualifier":"regioninfo","vlen":76,"tag":[],"timestamp":"1732466716106"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732466716106"}]},"ts":"1732466716106"} 2024-11-24T16:45:16,115 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-24T16:45:16,116 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=testBlockEvictionOnRegionMove1732466716044 execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-24T16:45:16,119 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testBlockEvictionOnRegionMove1732466716044","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732466716116"}]},"ts":"1732466716116"} 2024-11-24T16:45:16,123 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=testBlockEvictionOnRegionMove1732466716044, state=ENABLING in hbase:meta 2024-11-24T16:45:16,124 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(204): Hosts are {e335232f7491=0} racks are {/default-rack=0} 2024-11-24T16:45:16,125 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-11-24T16:45:16,125 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-11-24T16:45:16,125 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-11-24T16:45:16,125 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-11-24T16:45:16,125 INFO [PEWorker-3 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-11-24T16:45:16,125 INFO [PEWorker-3 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-11-24T16:45:16,125 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-11-24T16:45:16,126 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1732466716044, region=8d688d1c7c98ba065aee654c8fa6e77a, ASSIGN}] 2024-11-24T16:45:16,128 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1732466716044, region=8d688d1c7c98ba065aee654c8fa6e77a, ASSIGN 2024-11-24T16:45:16,129 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1732466716044, region=8d688d1c7c98ba065aee654c8fa6e77a, ASSIGN; state=OFFLINE, location=e335232f7491,45465,1732466713886; forceNewPlan=false, retain=false 2024-11-24T16:45:16,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45629 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-24T16:45:16,282 INFO [e335232f7491:45629 {}] balancer.BaseLoadBalancer(388): Reassigned 1 regions. 1 retained the pre-restart assignment. 2024-11-24T16:45:16,283 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=8d688d1c7c98ba065aee654c8fa6e77a, regionState=OPENING, regionLocation=e335232f7491,45465,1732466713886 2024-11-24T16:45:16,287 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-4-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1732466716044, region=8d688d1c7c98ba065aee654c8fa6e77a, ASSIGN because future has completed 2024-11-24T16:45:16,288 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 8d688d1c7c98ba065aee654c8fa6e77a, server=e335232f7491,45465,1732466713886}] 2024-11-24T16:45:16,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45629 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-24T16:45:16,444 INFO [RS_OPEN_REGION-regionserver/e335232f7491:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(132): Open testBlockEvictionOnRegionMove1732466716044,,1732466716067.8d688d1c7c98ba065aee654c8fa6e77a. 2024-11-24T16:45:16,445 DEBUG [RS_OPEN_REGION-regionserver/e335232f7491:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7752): Opening region: {ENCODED => 8d688d1c7c98ba065aee654c8fa6e77a, NAME => 'testBlockEvictionOnRegionMove1732466716044,,1732466716067.8d688d1c7c98ba065aee654c8fa6e77a.', STARTKEY => '', ENDKEY => ''} 2024-11-24T16:45:16,445 DEBUG [RS_OPEN_REGION-regionserver/e335232f7491:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testBlockEvictionOnRegionMove1732466716044 8d688d1c7c98ba065aee654c8fa6e77a 2024-11-24T16:45:16,445 DEBUG [RS_OPEN_REGION-regionserver/e335232f7491:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(898): Instantiated testBlockEvictionOnRegionMove1732466716044,,1732466716067.8d688d1c7c98ba065aee654c8fa6e77a.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T16:45:16,445 DEBUG [RS_OPEN_REGION-regionserver/e335232f7491:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7794): checking encryption for 8d688d1c7c98ba065aee654c8fa6e77a 2024-11-24T16:45:16,446 DEBUG [RS_OPEN_REGION-regionserver/e335232f7491:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7797): checking classloading for 8d688d1c7c98ba065aee654c8fa6e77a 2024-11-24T16:45:16,446 INFO [StoreOpener-8d688d1c7c98ba065aee654c8fa6e77a-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family family of region 8d688d1c7c98ba065aee654c8fa6e77a 2024-11-24T16:45:16,450 INFO [StoreOpener-8d688d1c7c98ba065aee654c8fa6e77a-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 8d688d1c7c98ba065aee654c8fa6e77a columnFamilyName family 2024-11-24T16:45:16,450 DEBUG [StoreOpener-8d688d1c7c98ba065aee654c8fa6e77a-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:45:16,451 INFO [StoreOpener-8d688d1c7c98ba065aee654c8fa6e77a-1 {}] regionserver.HStore(327): Store=8d688d1c7c98ba065aee654c8fa6e77a/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T16:45:16,451 DEBUG [RS_OPEN_REGION-regionserver/e335232f7491:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1038): replaying wal for 8d688d1c7c98ba065aee654c8fa6e77a 2024-11-24T16:45:16,452 DEBUG [RS_OPEN_REGION-regionserver/e335232f7491:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9797cf4c-7cf0-a873-8ba7-f30267b4a2f7/39f4fdd4-f188-1d18-4286-5b0c4dc27cad/data/default/testBlockEvictionOnRegionMove1732466716044/8d688d1c7c98ba065aee654c8fa6e77a 2024-11-24T16:45:16,452 DEBUG [RS_OPEN_REGION-regionserver/e335232f7491:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9797cf4c-7cf0-a873-8ba7-f30267b4a2f7/39f4fdd4-f188-1d18-4286-5b0c4dc27cad/data/default/testBlockEvictionOnRegionMove1732466716044/8d688d1c7c98ba065aee654c8fa6e77a 2024-11-24T16:45:16,452 DEBUG [RS_OPEN_REGION-regionserver/e335232f7491:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1048): stopping wal replay for 8d688d1c7c98ba065aee654c8fa6e77a 2024-11-24T16:45:16,452 DEBUG [RS_OPEN_REGION-regionserver/e335232f7491:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1060): Cleaning up temporary data for 8d688d1c7c98ba065aee654c8fa6e77a 2024-11-24T16:45:16,453 DEBUG [RS_OPEN_REGION-regionserver/e335232f7491:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1093): writing seq id for 8d688d1c7c98ba065aee654c8fa6e77a 2024-11-24T16:45:16,460 DEBUG [RS_OPEN_REGION-regionserver/e335232f7491:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9797cf4c-7cf0-a873-8ba7-f30267b4a2f7/39f4fdd4-f188-1d18-4286-5b0c4dc27cad/data/default/testBlockEvictionOnRegionMove1732466716044/8d688d1c7c98ba065aee654c8fa6e77a/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-24T16:45:16,461 INFO [RS_OPEN_REGION-regionserver/e335232f7491:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1114): Opened 8d688d1c7c98ba065aee654c8fa6e77a; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=65508806, jitterRate=-0.023842722177505493}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-24T16:45:16,461 DEBUG [RS_OPEN_REGION-regionserver/e335232f7491:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 8d688d1c7c98ba065aee654c8fa6e77a 2024-11-24T16:45:16,462 DEBUG [RS_OPEN_REGION-regionserver/e335232f7491:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1006): Region open journal for 8d688d1c7c98ba065aee654c8fa6e77a: Running coprocessor pre-open hook at 1732466716446Writing region info on filesystem at 1732466716446Initializing all the Stores at 1732466716446Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732466716446Cleaning up temporary data from old regions at 1732466716452 (+6 ms)Running coprocessor post-open hooks at 1732466716461 (+9 ms)Region opened successfully at 1732466716461 2024-11-24T16:45:16,463 INFO [RS_OPEN_REGION-regionserver/e335232f7491:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2236): Post open deploy tasks for testBlockEvictionOnRegionMove1732466716044,,1732466716067.8d688d1c7c98ba065aee654c8fa6e77a., pid=6, masterSystemTime=1732466716441 2024-11-24T16:45:16,466 DEBUG [RS_OPEN_REGION-regionserver/e335232f7491:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2266): Finished post open deploy task for testBlockEvictionOnRegionMove1732466716044,,1732466716067.8d688d1c7c98ba065aee654c8fa6e77a. 2024-11-24T16:45:16,466 INFO [RS_OPEN_REGION-regionserver/e335232f7491:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(153): Opened testBlockEvictionOnRegionMove1732466716044,,1732466716067.8d688d1c7c98ba065aee654c8fa6e77a. 2024-11-24T16:45:16,466 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=8d688d1c7c98ba065aee654c8fa6e77a, regionState=OPEN, openSeqNum=2, regionLocation=e335232f7491,45465,1732466713886 2024-11-24T16:45:16,469 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-4-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 8d688d1c7c98ba065aee654c8fa6e77a, server=e335232f7491,45465,1732466713886 because future has completed 2024-11-24T16:45:16,473 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=6, resume processing ppid=5 2024-11-24T16:45:16,473 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure 8d688d1c7c98ba065aee654c8fa6e77a, server=e335232f7491,45465,1732466713886 in 182 msec 2024-11-24T16:45:16,475 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=5, resume processing ppid=4 2024-11-24T16:45:16,475 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1732466716044, region=8d688d1c7c98ba065aee654c8fa6e77a, ASSIGN in 347 msec 2024-11-24T16:45:16,476 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=testBlockEvictionOnRegionMove1732466716044 execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-24T16:45:16,476 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testBlockEvictionOnRegionMove1732466716044","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732466716476"}]},"ts":"1732466716476"} 2024-11-24T16:45:16,479 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=testBlockEvictionOnRegionMove1732466716044, state=ENABLED in hbase:meta 2024-11-24T16:45:16,480 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=testBlockEvictionOnRegionMove1732466716044 execute state=CREATE_TABLE_POST_OPERATION 2024-11-24T16:45:16,482 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=testBlockEvictionOnRegionMove1732466716044 in 406 msec 2024-11-24T16:45:16,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45629 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-24T16:45:16,719 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table testBlockEvictionOnRegionMove1732466716044 get assigned. Timeout = 60000ms 2024-11-24T16:45:16,719 INFO [RPCClient-NioEventLoopGroup-5-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:testBlockEvictionOnRegionMove1732466716044 completed 2024-11-24T16:45:16,722 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-24T16:45:16,727 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table testBlockEvictionOnRegionMove1732466716044 assigned to meta. Checking AM states. 2024-11-24T16:45:16,727 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-24T16:45:16,728 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table testBlockEvictionOnRegionMove1732466716044 assigned. 2024-11-24T16:45:16,736 DEBUG [RPCClient-NioEventLoopGroup-5-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testBlockEvictionOnRegionMove1732466716044', row='row1', locateType=CURRENT is [region=testBlockEvictionOnRegionMove1732466716044,,1732466716067.8d688d1c7c98ba065aee654c8fa6e77a., hostname=e335232f7491,45465,1732466713886, seqNum=2] 2024-11-24T16:45:16,747 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 8d688d1c7c98ba065aee654c8fa6e77a 1/1 column families, dataSize=78 B heapSize=480 B 2024-11-24T16:45:16,772 DEBUG [Time-limited test {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on b4ec7c78b33d4365b798fb9a751d2936 2024-11-24T16:45:16,809 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9797cf4c-7cf0-a873-8ba7-f30267b4a2f7/39f4fdd4-f188-1d18-4286-5b0c4dc27cad/data/default/testBlockEvictionOnRegionMove1732466716044/8d688d1c7c98ba065aee654c8fa6e77a/.tmp/family/b4ec7c78b33d4365b798fb9a751d2936 is 43, key is row1/family:qf1/1/Put/seqid=0 2024-11-24T16:45:16,814 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=78 B at sequenceid=6 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9797cf4c-7cf0-a873-8ba7-f30267b4a2f7/39f4fdd4-f188-1d18-4286-5b0c4dc27cad/data/default/testBlockEvictionOnRegionMove1732466716044/8d688d1c7c98ba065aee654c8fa6e77a/.tmp/family/b4ec7c78b33d4365b798fb9a751d2936 2024-11-24T16:45:16,850 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9797cf4c-7cf0-a873-8ba7-f30267b4a2f7/39f4fdd4-f188-1d18-4286-5b0c4dc27cad/data/default/testBlockEvictionOnRegionMove1732466716044/8d688d1c7c98ba065aee654c8fa6e77a/.tmp/family/b4ec7c78b33d4365b798fb9a751d2936 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9797cf4c-7cf0-a873-8ba7-f30267b4a2f7/39f4fdd4-f188-1d18-4286-5b0c4dc27cad/data/default/testBlockEvictionOnRegionMove1732466716044/8d688d1c7c98ba065aee654c8fa6e77a/family/b4ec7c78b33d4365b798fb9a751d2936 2024-11-24T16:45:16,852 DEBUG [Time-limited test {}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9797cf4c-7cf0-a873-8ba7-f30267b4a2f7/39f4fdd4-f188-1d18-4286-5b0c4dc27cad/data/default/testBlockEvictionOnRegionMove1732466716044/8d688d1c7c98ba065aee654c8fa6e77a/family/b4ec7c78b33d4365b798fb9a751d2936, delay=933 ms 2024-11-24T16:45:16,853 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9797cf4c-7cf0-a873-8ba7-f30267b4a2f7/39f4fdd4-f188-1d18-4286-5b0c4dc27cad/data/default/testBlockEvictionOnRegionMove1732466716044/8d688d1c7c98ba065aee654c8fa6e77a/family/b4ec7c78b33d4365b798fb9a751d2936, entries=2, sequenceid=6, filesize=4.9 K 2024-11-24T16:45:16,859 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~78 B/78, heapSize ~464 B/464, currentSize=0 B/0 for 8d688d1c7c98ba065aee654c8fa6e77a in 111ms, sequenceid=6, compaction requested=false 2024-11-24T16:45:16,860 DEBUG [Time-limited test {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'testBlockEvictionOnRegionMove1732466716044' 2024-11-24T16:45:16,862 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for 8d688d1c7c98ba065aee654c8fa6e77a: 2024-11-24T16:45:17,787 DEBUG [hfile-prefetch-1732466716852 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key b4ec7c78b33d4365b798fb9a751d2936_0. 2024-11-24T16:45:17,791 DEBUG [hfile-prefetch-1732466716852 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key b4ec7c78b33d4365b798fb9a751d2936_117. 2024-11-24T16:45:17,792 DEBUG [hfile-prefetch-1732466716852 {}] bucket.BucketCache(2281): Notifying caching completed for file file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9797cf4c-7cf0-a873-8ba7-f30267b4a2f7/39f4fdd4-f188-1d18-4286-5b0c4dc27cad/data/default/testBlockEvictionOnRegionMove1732466716044/8d688d1c7c98ba065aee654c8fa6e77a/family/b4ec7c78b33d4365b798fb9a751d2936, with total blocks 2, and data blocks 1 2024-11-24T16:45:17,792 DEBUG [hfile-prefetch-1732466716852 {}] bucket.BucketCache(2285): iterating over 0 entries in the backing map 2024-11-24T16:45:17,793 DEBUG [hfile-prefetch-1732466716852 {}] bucket.BucketCache(2308): Prefetch executor completed for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9797cf4c-7cf0-a873-8ba7-f30267b4a2f7/39f4fdd4-f188-1d18-4286-5b0c4dc27cad/data/default/testBlockEvictionOnRegionMove1732466716044/8d688d1c7c98ba065aee654c8fa6e77a/family/b4ec7c78b33d4365b798fb9a751d2936, but only 0 data blocks were cached. Total data blocks for file: 1. Checking for blocks pending cache in cache writer queue. 2024-11-24T16:45:17,793 DEBUG [hfile-prefetch-1732466716852 {}] bucket.BucketCache(2318): There are still blocks pending caching for file file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9797cf4c-7cf0-a873-8ba7-f30267b4a2f7/39f4fdd4-f188-1d18-4286-5b0c4dc27cad/data/default/testBlockEvictionOnRegionMove1732466716044/8d688d1c7c98ba065aee654c8fa6e77a/family/b4ec7c78b33d4365b798fb9a751d2936. Will sleep 100ms and try the verification again. 2024-11-24T16:45:17,848 DEBUG [bucket-cache-persister {}] bucket.BucketCachePersister(47): Cache is inconsistent, persisting to disk 2024-11-24T16:45:17,848 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1487): Thread bucket-cache-persister started persisting bucket cache to file 2024-11-24T16:45:17,849 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1494): Persist in new chunked persistence format. 2024-11-24T16:45:17,849 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1673): persistToFile: before persisting backing map size: 0, fullycachedFiles size: 0, chunkSize: 10000 2024-11-24T16:45:17,865 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1680): persistToFile: after persisting backing map size: 0, fullycachedFiles size: 0 2024-11-24T16:45:17,865 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1498): PersistToFile: after persisting backing map size: 0, fullycachedFiles size: 0, file name: bucket.persistence1732466717849 2024-11-24T16:45:17,865 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1509): Thread bucket-cache-persister finished persisting bucket cache to file, renaming 2024-11-24T16:45:17,872 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45629 {}] master.HMaster(2410): Client=jenkins//172.17.0.2 move hri=8d688d1c7c98ba065aee654c8fa6e77a, source=e335232f7491,45465,1732466713886, destination=e335232f7491,39717,1732466714030, warming up region on e335232f7491,39717,1732466714030 2024-11-24T16:45:17,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45629 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-24T16:45:17,875 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45629 {}] master.HMaster(2414): Client=jenkins//172.17.0.2 move hri=8d688d1c7c98ba065aee654c8fa6e77a, source=e335232f7491,45465,1732466713886, destination=e335232f7491,39717,1732466714030, running balancer 2024-11-24T16:45:17,876 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-4-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35971, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-24T16:45:17,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45629 {}] procedure2.ProcedureExecutor(1139): Stored pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1732466716044, region=8d688d1c7c98ba065aee654c8fa6e77a, REOPEN/MOVE 2024-11-24T16:45:17,877 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1732466716044, region=8d688d1c7c98ba065aee654c8fa6e77a, REOPEN/MOVE 2024-11-24T16:45:17,878 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39717 {}] regionserver.RSRpcServices(2066): Warmup testBlockEvictionOnRegionMove1732466716044,,1732466716067.8d688d1c7c98ba065aee654c8fa6e77a. 2024-11-24T16:45:17,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39717 {}] regionserver.HRegion(7855): Warmup {ENCODED => 8d688d1c7c98ba065aee654c8fa6e77a, NAME => 'testBlockEvictionOnRegionMove1732466716044,,1732466716067.8d688d1c7c98ba065aee654c8fa6e77a.', STARTKEY => '', ENDKEY => ''} 2024-11-24T16:45:17,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39717 {}] regionserver.HRegion(898): Instantiated testBlockEvictionOnRegionMove1732466716044,,1732466716067.8d688d1c7c98ba065aee654c8fa6e77a.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T16:45:17,879 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=8d688d1c7c98ba065aee654c8fa6e77a, regionState=CLOSING, regionLocation=e335232f7491,45465,1732466713886 2024-11-24T16:45:17,879 INFO [StoreOpener-8d688d1c7c98ba065aee654c8fa6e77a-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family family of region 8d688d1c7c98ba065aee654c8fa6e77a 2024-11-24T16:45:17,880 INFO [StoreOpener-8d688d1c7c98ba065aee654c8fa6e77a-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 8d688d1c7c98ba065aee654c8fa6e77a columnFamilyName family 2024-11-24T16:45:17,880 DEBUG [StoreOpener-8d688d1c7c98ba065aee654c8fa6e77a-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:45:17,882 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-4-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1732466716044, region=8d688d1c7c98ba065aee654c8fa6e77a, REOPEN/MOVE because future has completed 2024-11-24T16:45:17,883 DEBUG [PEWorker-3 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: true 2024-11-24T16:45:17,883 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=8, ppid=7, state=RUNNABLE, hasLock=false; CloseRegionProcedure 8d688d1c7c98ba065aee654c8fa6e77a, server=e335232f7491,45465,1732466713886}] 2024-11-24T16:45:17,884 DEBUG [StoreOpener-8d688d1c7c98ba065aee654c8fa6e77a-1 {}] regionserver.StoreEngine(278): loaded file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9797cf4c-7cf0-a873-8ba7-f30267b4a2f7/39f4fdd4-f188-1d18-4286-5b0c4dc27cad/data/default/testBlockEvictionOnRegionMove1732466716044/8d688d1c7c98ba065aee654c8fa6e77a/family/b4ec7c78b33d4365b798fb9a751d2936 2024-11-24T16:45:17,884 INFO [StoreOpener-8d688d1c7c98ba065aee654c8fa6e77a-1 {}] regionserver.HStore(327): Store=8d688d1c7c98ba065aee654c8fa6e77a/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T16:45:17,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39717 {}] regionserver.HRegion(1722): Closing 8d688d1c7c98ba065aee654c8fa6e77a, disabling compactions & flushes 2024-11-24T16:45:17,884 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39717 {}] regionserver.HRegion(1755): Closing region testBlockEvictionOnRegionMove1732466716044,,1732466716067.8d688d1c7c98ba065aee654c8fa6e77a. 2024-11-24T16:45:17,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39717 {}] regionserver.HRegion(1776): Time limited wait for close lock on testBlockEvictionOnRegionMove1732466716044,,1732466716067.8d688d1c7c98ba065aee654c8fa6e77a. 2024-11-24T16:45:17,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39717 {}] regionserver.HRegion(1843): Acquired close lock on testBlockEvictionOnRegionMove1732466716044,,1732466716067.8d688d1c7c98ba065aee654c8fa6e77a. after waiting 0 ms 2024-11-24T16:45:17,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39717 {}] regionserver.HRegion(1853): Updates disabled for region testBlockEvictionOnRegionMove1732466716044,,1732466716067.8d688d1c7c98ba065aee654c8fa6e77a. 2024-11-24T16:45:17,887 DEBUG [StoreFileCloser-8d688d1c7c98ba065aee654c8fa6e77a-family-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9797cf4c-7cf0-a873-8ba7-f30267b4a2f7/39f4fdd4-f188-1d18-4286-5b0c4dc27cad/data/default/testBlockEvictionOnRegionMove1732466716044/8d688d1c7c98ba065aee654c8fa6e77a/family/b4ec7c78b33d4365b798fb9a751d2936 2024-11-24T16:45:17,887 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39717 {}] regionserver.HRegion(1973): Closed testBlockEvictionOnRegionMove1732466716044,,1732466716067.8d688d1c7c98ba065aee654c8fa6e77a. 2024-11-24T16:45:17,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39717 {}] regionserver.HRegion(1676): Region close journal for 8d688d1c7c98ba065aee654c8fa6e77a: Waiting for close lock at 1732466717884Disabling compacts and flushes for region at 1732466717884Disabling writes for close at 1732466717884Writing region close event to WAL at 1732466717887 (+3 ms)Closed at 1732466717887 2024-11-24T16:45:17,889 WARN [hfile-prefetch-1732466716852 {}] hfile.HFilePreadReader$1(158): Prefetch path=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9797cf4c-7cf0-a873-8ba7-f30267b4a2f7/39f4fdd4-f188-1d18-4286-5b0c4dc27cad/data/default/testBlockEvictionOnRegionMove1732466716044/8d688d1c7c98ba065aee654c8fa6e77a/family/b4ec7c78b33d4365b798fb9a751d2936, offset=158, end=158 java.lang.RuntimeException: java.lang.InterruptedException: sleep interrupted at org.apache.hadoop.hbase.io.hfile.bucket.BucketCache.notifyFileCachingCompleted(BucketCache.java:2337) ~[classes/:?] at org.apache.hadoop.hbase.io.hfile.CombinedBlockCache.notifyFileCachingCompleted(CombinedBlockCache.java:469) ~[classes/:?] at org.apache.hadoop.hbase.io.hfile.HFilePreadReader$1.run(HFilePreadReader.java:148) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.lambda$tracedRunnable$2(TraceUtil.java:155) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:539) ~[?:?] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:304) ~[?:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCache.notifyFileCachingCompleted(BucketCache.java:2320) ~[classes/:?] ... 9 more 2024-11-24T16:45:17,890 DEBUG [hfile-prefetch-1732466716852 {}] hfile.PrefetchExecutor(121): Prefetch completed for b4ec7c78b33d4365b798fb9a751d2936 2024-11-24T16:45:17,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45629 {}] procedure.ProcedureSyncWait(219): waitFor pid=7 2024-11-24T16:45:18,041 INFO [RS_CLOSE_REGION-regionserver/e335232f7491:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] handler.UnassignRegionHandler(122): Close 8d688d1c7c98ba065aee654c8fa6e77a 2024-11-24T16:45:18,041 DEBUG [RS_CLOSE_REGION-regionserver/e335232f7491:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: true 2024-11-24T16:45:18,042 DEBUG [RS_CLOSE_REGION-regionserver/e335232f7491:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1722): Closing 8d688d1c7c98ba065aee654c8fa6e77a, disabling compactions & flushes 2024-11-24T16:45:18,042 INFO [RS_CLOSE_REGION-regionserver/e335232f7491:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1755): Closing region testBlockEvictionOnRegionMove1732466716044,,1732466716067.8d688d1c7c98ba065aee654c8fa6e77a. 2024-11-24T16:45:18,042 DEBUG [RS_CLOSE_REGION-regionserver/e335232f7491:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1776): Time limited wait for close lock on testBlockEvictionOnRegionMove1732466716044,,1732466716067.8d688d1c7c98ba065aee654c8fa6e77a. 2024-11-24T16:45:18,042 DEBUG [RS_CLOSE_REGION-regionserver/e335232f7491:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1843): Acquired close lock on testBlockEvictionOnRegionMove1732466716044,,1732466716067.8d688d1c7c98ba065aee654c8fa6e77a. after waiting 0 ms 2024-11-24T16:45:18,042 DEBUG [RS_CLOSE_REGION-regionserver/e335232f7491:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1853): Updates disabled for region testBlockEvictionOnRegionMove1732466716044,,1732466716067.8d688d1c7c98ba065aee654c8fa6e77a. 2024-11-24T16:45:18,044 DEBUG [StoreFileCloser-8d688d1c7c98ba065aee654c8fa6e77a-family-1 {}] bucket.BucketCache(1863): found 1 blocks for file b4ec7c78b33d4365b798fb9a751d2936, starting offset: 0, end offset: 9223372036854775807 2024-11-24T16:45:18,045 DEBUG [StoreFileCloser-8d688d1c7c98ba065aee654c8fa6e77a-family-1 {}] bucket.BucketCache(856): removed key b4ec7c78b33d4365b798fb9a751d2936_0 from back map with offset lock 2094080 in the evict process 2024-11-24T16:45:18,051 DEBUG [RS_CLOSE_REGION-regionserver/e335232f7491:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9797cf4c-7cf0-a873-8ba7-f30267b4a2f7/39f4fdd4-f188-1d18-4286-5b0c4dc27cad/data/default/testBlockEvictionOnRegionMove1732466716044/8d688d1c7c98ba065aee654c8fa6e77a/recovered.edits/9.seqid, newMaxSeqId=9, maxSeqId=1 2024-11-24T16:45:18,052 INFO [RS_CLOSE_REGION-regionserver/e335232f7491:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1973): Closed testBlockEvictionOnRegionMove1732466716044,,1732466716067.8d688d1c7c98ba065aee654c8fa6e77a. 2024-11-24T16:45:18,052 DEBUG [RS_CLOSE_REGION-regionserver/e335232f7491:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1676): Region close journal for 8d688d1c7c98ba065aee654c8fa6e77a: Waiting for close lock at 1732466718042Running coprocessor pre-close hooks at 1732466718042Disabling compacts and flushes for region at 1732466718042Disabling writes for close at 1732466718042Writing region close event to WAL at 1732466718046 (+4 ms)Running coprocessor post-close hooks at 1732466718051 (+5 ms)Closed at 1732466718052 (+1 ms) 2024-11-24T16:45:18,053 INFO [RS_CLOSE_REGION-regionserver/e335232f7491:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegionServer(3302): Adding 8d688d1c7c98ba065aee654c8fa6e77a move to e335232f7491,39717,1732466714030 record at close sequenceid=6 2024-11-24T16:45:18,055 INFO [RS_CLOSE_REGION-regionserver/e335232f7491:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] handler.UnassignRegionHandler(157): Closed 8d688d1c7c98ba065aee654c8fa6e77a 2024-11-24T16:45:18,056 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=8d688d1c7c98ba065aee654c8fa6e77a, regionState=CLOSED 2024-11-24T16:45:18,058 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-4-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=8, ppid=7, state=RUNNABLE, hasLock=false; CloseRegionProcedure 8d688d1c7c98ba065aee654c8fa6e77a, server=e335232f7491,45465,1732466713886 because future has completed 2024-11-24T16:45:18,061 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=8, resume processing ppid=7 2024-11-24T16:45:18,062 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=8, ppid=7, state=SUCCESS, hasLock=false; CloseRegionProcedure 8d688d1c7c98ba065aee654c8fa6e77a, server=e335232f7491,45465,1732466713886 in 177 msec 2024-11-24T16:45:18,062 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1732466716044, region=8d688d1c7c98ba065aee654c8fa6e77a, REOPEN/MOVE; state=CLOSED, location=e335232f7491,39717,1732466714030; forceNewPlan=false, retain=false 2024-11-24T16:45:18,065 DEBUG [bucket-cache-persister {}] bucket.BucketCachePersister(47): Cache is inconsistent, persisting to disk 2024-11-24T16:45:18,065 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1487): Thread bucket-cache-persister started persisting bucket cache to file 2024-11-24T16:45:18,065 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1494): Persist in new chunked persistence format. 2024-11-24T16:45:18,065 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1673): persistToFile: before persisting backing map size: 0, fullycachedFiles size: 0, chunkSize: 10000 2024-11-24T16:45:18,069 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1680): persistToFile: after persisting backing map size: 0, fullycachedFiles size: 0 2024-11-24T16:45:18,069 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1498): PersistToFile: after persisting backing map size: 0, fullycachedFiles size: 0, file name: bucket.persistence1732466718065 2024-11-24T16:45:18,069 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1509): Thread bucket-cache-persister finished persisting bucket cache to file, renaming 2024-11-24T16:45:18,213 INFO [e335232f7491:45629 {}] balancer.BaseLoadBalancer(388): Reassigned 1 regions. 1 retained the pre-restart assignment. 2024-11-24T16:45:18,213 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=8d688d1c7c98ba065aee654c8fa6e77a, regionState=OPENING, regionLocation=e335232f7491,39717,1732466714030 2024-11-24T16:45:18,217 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-4-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1732466716044, region=8d688d1c7c98ba065aee654c8fa6e77a, REOPEN/MOVE because future has completed 2024-11-24T16:45:18,218 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=9, ppid=7, state=RUNNABLE, hasLock=false; OpenRegionProcedure 8d688d1c7c98ba065aee654c8fa6e77a, server=e335232f7491,39717,1732466714030}] 2024-11-24T16:45:18,372 INFO [RS_OPEN_REGION-regionserver/e335232f7491:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] handler.AssignRegionHandler(132): Open testBlockEvictionOnRegionMove1732466716044,,1732466716067.8d688d1c7c98ba065aee654c8fa6e77a. 2024-11-24T16:45:18,373 DEBUG [RS_OPEN_REGION-regionserver/e335232f7491:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(7752): Opening region: {ENCODED => 8d688d1c7c98ba065aee654c8fa6e77a, NAME => 'testBlockEvictionOnRegionMove1732466716044,,1732466716067.8d688d1c7c98ba065aee654c8fa6e77a.', STARTKEY => '', ENDKEY => ''} 2024-11-24T16:45:18,373 DEBUG [RS_OPEN_REGION-regionserver/e335232f7491:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testBlockEvictionOnRegionMove1732466716044 8d688d1c7c98ba065aee654c8fa6e77a 2024-11-24T16:45:18,373 DEBUG [RS_OPEN_REGION-regionserver/e335232f7491:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(898): Instantiated testBlockEvictionOnRegionMove1732466716044,,1732466716067.8d688d1c7c98ba065aee654c8fa6e77a.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T16:45:18,374 DEBUG [RS_OPEN_REGION-regionserver/e335232f7491:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(7794): checking encryption for 8d688d1c7c98ba065aee654c8fa6e77a 2024-11-24T16:45:18,374 DEBUG [RS_OPEN_REGION-regionserver/e335232f7491:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(7797): checking classloading for 8d688d1c7c98ba065aee654c8fa6e77a 2024-11-24T16:45:18,375 INFO [StoreOpener-8d688d1c7c98ba065aee654c8fa6e77a-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family family of region 8d688d1c7c98ba065aee654c8fa6e77a 2024-11-24T16:45:18,376 INFO [StoreOpener-8d688d1c7c98ba065aee654c8fa6e77a-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 8d688d1c7c98ba065aee654c8fa6e77a columnFamilyName family 2024-11-24T16:45:18,376 DEBUG [StoreOpener-8d688d1c7c98ba065aee654c8fa6e77a-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:45:18,377 DEBUG [StoreFileOpener-8d688d1c7c98ba065aee654c8fa6e77a-family-1 {}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9797cf4c-7cf0-a873-8ba7-f30267b4a2f7/39f4fdd4-f188-1d18-4286-5b0c4dc27cad/data/default/testBlockEvictionOnRegionMove1732466716044/8d688d1c7c98ba065aee654c8fa6e77a/family/b4ec7c78b33d4365b798fb9a751d2936, delay=974 ms 2024-11-24T16:45:18,379 DEBUG [StoreOpener-8d688d1c7c98ba065aee654c8fa6e77a-1 {}] regionserver.StoreEngine(278): loaded file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9797cf4c-7cf0-a873-8ba7-f30267b4a2f7/39f4fdd4-f188-1d18-4286-5b0c4dc27cad/data/default/testBlockEvictionOnRegionMove1732466716044/8d688d1c7c98ba065aee654c8fa6e77a/family/b4ec7c78b33d4365b798fb9a751d2936 2024-11-24T16:45:18,379 INFO [StoreOpener-8d688d1c7c98ba065aee654c8fa6e77a-1 {}] regionserver.HStore(327): Store=8d688d1c7c98ba065aee654c8fa6e77a/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T16:45:18,379 DEBUG [RS_OPEN_REGION-regionserver/e335232f7491:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1038): replaying wal for 8d688d1c7c98ba065aee654c8fa6e77a 2024-11-24T16:45:18,380 DEBUG [RS_OPEN_REGION-regionserver/e335232f7491:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9797cf4c-7cf0-a873-8ba7-f30267b4a2f7/39f4fdd4-f188-1d18-4286-5b0c4dc27cad/data/default/testBlockEvictionOnRegionMove1732466716044/8d688d1c7c98ba065aee654c8fa6e77a 2024-11-24T16:45:18,381 DEBUG [RS_OPEN_REGION-regionserver/e335232f7491:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9797cf4c-7cf0-a873-8ba7-f30267b4a2f7/39f4fdd4-f188-1d18-4286-5b0c4dc27cad/data/default/testBlockEvictionOnRegionMove1732466716044/8d688d1c7c98ba065aee654c8fa6e77a 2024-11-24T16:45:18,381 DEBUG [RS_OPEN_REGION-regionserver/e335232f7491:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1048): stopping wal replay for 8d688d1c7c98ba065aee654c8fa6e77a 2024-11-24T16:45:18,381 DEBUG [RS_OPEN_REGION-regionserver/e335232f7491:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1060): Cleaning up temporary data for 8d688d1c7c98ba065aee654c8fa6e77a 2024-11-24T16:45:18,385 DEBUG [RS_OPEN_REGION-regionserver/e335232f7491:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1093): writing seq id for 8d688d1c7c98ba065aee654c8fa6e77a 2024-11-24T16:45:18,386 INFO [RS_OPEN_REGION-regionserver/e335232f7491:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1114): Opened 8d688d1c7c98ba065aee654c8fa6e77a; next sequenceid=10; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=75218016, jitterRate=0.12083578109741211}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-24T16:45:18,386 DEBUG [RS_OPEN_REGION-regionserver/e335232f7491:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 8d688d1c7c98ba065aee654c8fa6e77a 2024-11-24T16:45:18,387 DEBUG [RS_OPEN_REGION-regionserver/e335232f7491:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1006): Region open journal for 8d688d1c7c98ba065aee654c8fa6e77a: Running coprocessor pre-open hook at 1732466718374Writing region info on filesystem at 1732466718374Initializing all the Stores at 1732466718374Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732466718374Cleaning up temporary data from old regions at 1732466718381 (+7 ms)Running coprocessor post-open hooks at 1732466718386 (+5 ms)Region opened successfully at 1732466718387 (+1 ms) 2024-11-24T16:45:18,388 INFO [RS_OPEN_REGION-regionserver/e335232f7491:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegionServer(2236): Post open deploy tasks for testBlockEvictionOnRegionMove1732466716044,,1732466716067.8d688d1c7c98ba065aee654c8fa6e77a., pid=9, masterSystemTime=1732466718370 2024-11-24T16:45:18,390 DEBUG [RS_OPEN_REGION-regionserver/e335232f7491:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegionServer(2266): Finished post open deploy task for testBlockEvictionOnRegionMove1732466716044,,1732466716067.8d688d1c7c98ba065aee654c8fa6e77a. 2024-11-24T16:45:18,390 INFO [RS_OPEN_REGION-regionserver/e335232f7491:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] handler.AssignRegionHandler(153): Opened testBlockEvictionOnRegionMove1732466716044,,1732466716067.8d688d1c7c98ba065aee654c8fa6e77a. 2024-11-24T16:45:18,391 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=8d688d1c7c98ba065aee654c8fa6e77a, regionState=OPEN, openSeqNum=10, regionLocation=e335232f7491,39717,1732466714030 2024-11-24T16:45:18,393 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-4-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=9, ppid=7, state=RUNNABLE, hasLock=false; OpenRegionProcedure 8d688d1c7c98ba065aee654c8fa6e77a, server=e335232f7491,39717,1732466714030 because future has completed 2024-11-24T16:45:18,396 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=9, resume processing ppid=7 2024-11-24T16:45:18,396 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=9, ppid=7, state=SUCCESS, hasLock=false; OpenRegionProcedure 8d688d1c7c98ba065aee654c8fa6e77a, server=e335232f7491,39717,1732466714030 in 177 msec 2024-11-24T16:45:18,397 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=7, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1732466716044, region=8d688d1c7c98ba065aee654c8fa6e77a, REOPEN/MOVE in 521 msec 2024-11-24T16:45:18,430 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster 2024-11-24T16:45:18,430 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-24T16:45:18,431 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79) at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611) at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.io.hfile.TestBlockEvictionOnRegionMovement.tearDown(TestBlockEvictionOnRegionMovement.java:174) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-24T16:45:18,435 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T16:45:18,436 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T16:45:18,436 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-24T16:45:18,436 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2024-11-24T16:45:18,436 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=632300009, stopped=false 2024-11-24T16:45:18,437 INFO [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=e335232f7491,45629,1732466713116 2024-11-24T16:45:18,439 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45629-0x1014ade0b8c0000, quorum=127.0.0.1:60963, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-24T16:45:18,439 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39717-0x1014ade0b8c0002, quorum=127.0.0.1:60963, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-24T16:45:18,439 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45465-0x1014ade0b8c0001, quorum=127.0.0.1:60963, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-24T16:45:18,439 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45629-0x1014ade0b8c0000, quorum=127.0.0.1:60963, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T16:45:18,439 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39717-0x1014ade0b8c0002, quorum=127.0.0.1:60963, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T16:45:18,439 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45465-0x1014ade0b8c0001, quorum=127.0.0.1:60963, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T16:45:18,439 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-24T16:45:18,439 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-24T16:45:18,440 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277) at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265) at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416) at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.io.hfile.TestBlockEvictionOnRegionMovement.tearDown(TestBlockEvictionOnRegionMovement.java:174) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-24T16:45:18,440 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T16:45:18,440 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:45465-0x1014ade0b8c0001, quorum=127.0.0.1:60963, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-24T16:45:18,440 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:45629-0x1014ade0b8c0000, quorum=127.0.0.1:60963, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-24T16:45:18,440 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:39717-0x1014ade0b8c0002, quorum=127.0.0.1:60963, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-24T16:45:18,441 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server 'e335232f7491,45465,1732466713886' ***** 2024-11-24T16:45:18,441 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-11-24T16:45:18,441 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server 'e335232f7491,39717,1732466714030' ***** 2024-11-24T16:45:18,441 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-11-24T16:45:18,441 INFO [RS:0;e335232f7491:45465 {}] bucket.BucketCache(1738): Disabling cache 2024-11-24T16:45:18,441 INFO [RS:0;e335232f7491:45465 {}] bucket.BucketCache(1762): Shutdown bucket cache: IO persistent=true; path to write=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9797cf4c-7cf0-a873-8ba7-f30267b4a2f7/bucket.persistence 2024-11-24T16:45:18,441 INFO [Time-limited test-BucketCacheWriter-0 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-0 exiting, cacheEnabled=false 2024-11-24T16:45:18,441 INFO [Time-limited test-BucketCacheWriter-1 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-1 exiting, cacheEnabled=false 2024-11-24T16:45:18,442 INFO [Time-limited test-BucketCacheWriter-2 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-2 exiting, cacheEnabled=false 2024-11-24T16:45:18,442 INFO [RS:1;e335232f7491:39717 {}] bucket.BucketCache(1738): Disabling cache 2024-11-24T16:45:18,442 INFO [RS:0;e335232f7491:45465 {}] bucket.BucketCache(1768): Shutting down cache persister thread. 2024-11-24T16:45:18,442 INFO [Time-limited test-BucketCacheWriter-0 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-0 exiting, cacheEnabled=false 2024-11-24T16:45:18,442 INFO [RS:1;e335232f7491:39717 {}] bucket.BucketCache(1762): Shutdown bucket cache: IO persistent=true; path to write=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9797cf4c-7cf0-a873-8ba7-f30267b4a2f7/bucket.persistence 2024-11-24T16:45:18,442 INFO [Time-limited test-BucketCacheWriter-1 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-1 exiting, cacheEnabled=false 2024-11-24T16:45:18,442 INFO [Time-limited test-BucketCacheWriter-2 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-2 exiting, cacheEnabled=false 2024-11-24T16:45:18,442 WARN [bucket-cache-persister {}] bucket.BucketCachePersister(64): Interrupting BucketCachePersister thread. java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCachePersister.run(BucketCachePersister.java:45) ~[classes/:?] 2024-11-24T16:45:18,442 INFO [RS:1;e335232f7491:39717 {}] bucket.BucketCache(1768): Shutting down cache persister thread. 2024-11-24T16:45:18,442 WARN [bucket-cache-persister {}] bucket.BucketCachePersister(64): Interrupting BucketCachePersister thread. java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCachePersister.run(BucketCachePersister.java:45) ~[classes/:?] 2024-11-24T16:45:18,452 DEBUG [RS:0;e335232f7491:45465 {}] bucket.BucketCache(1487): Thread RS:0;e335232f7491:45465 started persisting bucket cache to file 2024-11-24T16:45:18,452 DEBUG [RS:0;e335232f7491:45465 {}] bucket.BucketCache(1494): Persist in new chunked persistence format. 2024-11-24T16:45:18,452 DEBUG [RS:0;e335232f7491:45465 {}] bucket.BucketCache(1673): persistToFile: before persisting backing map size: 0, fullycachedFiles size: 0, chunkSize: 10000 2024-11-24T16:45:18,452 DEBUG [RS:1;e335232f7491:39717 {}] bucket.BucketCache(1487): Thread RS:1;e335232f7491:39717 started persisting bucket cache to file 2024-11-24T16:45:18,453 DEBUG [RS:1;e335232f7491:39717 {}] bucket.BucketCache(1494): Persist in new chunked persistence format. 2024-11-24T16:45:18,453 DEBUG [RS:1;e335232f7491:39717 {}] bucket.BucketCache(1673): persistToFile: before persisting backing map size: 0, fullycachedFiles size: 0, chunkSize: 10000 2024-11-24T16:45:18,456 DEBUG [RS:1;e335232f7491:39717 {}] bucket.BucketCache(1680): persistToFile: after persisting backing map size: 0, fullycachedFiles size: 0 2024-11-24T16:45:18,456 DEBUG [RS:1;e335232f7491:39717 {}] bucket.BucketCache(1498): PersistToFile: after persisting backing map size: 0, fullycachedFiles size: 0, file name: bucket.persistence1732466718452 2024-11-24T16:45:18,456 DEBUG [RS:1;e335232f7491:39717 {}] bucket.BucketCache(1509): Thread RS:1;e335232f7491:39717 finished persisting bucket cache to file, renaming 2024-11-24T16:45:18,457 DEBUG [RS:0;e335232f7491:45465 {}] bucket.BucketCache(1680): persistToFile: after persisting backing map size: 0, fullycachedFiles size: 0 2024-11-24T16:45:18,457 DEBUG [RS:0;e335232f7491:45465 {}] bucket.BucketCache(1498): PersistToFile: after persisting backing map size: 0, fullycachedFiles size: 0, file name: bucket.persistence1732466718452 2024-11-24T16:45:18,457 INFO [RS:1;e335232f7491:39717 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-24T16:45:18,457 DEBUG [RS:0;e335232f7491:45465 {}] bucket.BucketCache(1509): Thread RS:0;e335232f7491:45465 finished persisting bucket cache to file, renaming 2024-11-24T16:45:18,457 WARN [RS:0;e335232f7491:45465 {}] bucket.BucketCache(1512): Failed to commit cache persistent file. We might lose cached blocks if RS crashes/restarts before we successfully checkpoint again. 2024-11-24T16:45:18,457 INFO [RS:0;e335232f7491:45465 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-24T16:45:18,457 INFO [RS:1;e335232f7491:39717 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-24T16:45:18,457 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-24T16:45:18,457 INFO [RS:0;e335232f7491:45465 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-24T16:45:18,457 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-24T16:45:18,457 INFO [RS:0;e335232f7491:45465 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-24T16:45:18,457 INFO [RS:1;e335232f7491:39717 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-24T16:45:18,457 INFO [RS:0;e335232f7491:45465 {}] regionserver.HRegionServer(959): stopping server e335232f7491,45465,1732466713886 2024-11-24T16:45:18,457 INFO [RS:1;e335232f7491:39717 {}] regionserver.HRegionServer(3091): Received CLOSE for 8d688d1c7c98ba065aee654c8fa6e77a 2024-11-24T16:45:18,457 INFO [RS:0;e335232f7491:45465 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-24T16:45:18,458 INFO [RS:0;e335232f7491:45465 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;e335232f7491:45465. 2024-11-24T16:45:18,458 DEBUG [RS:0;e335232f7491:45465 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-24T16:45:18,458 DEBUG [RS:0;e335232f7491:45465 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T16:45:18,458 INFO [RS:1;e335232f7491:39717 {}] regionserver.HRegionServer(959): stopping server e335232f7491,39717,1732466714030 2024-11-24T16:45:18,458 INFO [RS:1;e335232f7491:39717 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-24T16:45:18,458 INFO [RS:1;e335232f7491:39717 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:1;e335232f7491:39717. 2024-11-24T16:45:18,458 INFO [RS:0;e335232f7491:45465 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-24T16:45:18,458 DEBUG [RS:1;e335232f7491:39717 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-24T16:45:18,458 INFO [RS:0;e335232f7491:45465 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-24T16:45:18,458 DEBUG [RS:1;e335232f7491:39717 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T16:45:18,458 INFO [RS:0;e335232f7491:45465 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-24T16:45:18,458 INFO [RS:0;e335232f7491:45465 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-11-24T16:45:18,458 INFO [RS_CLOSE_REGION-regionserver/e335232f7491:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1662): Closing region testBlockEvictionOnRegionMove1732466716044,,1732466716067.8d688d1c7c98ba065aee654c8fa6e77a. during a graceful stop, and cache persistence is on, so setting evict on close to false. 2024-11-24T16:45:18,458 INFO [RS:1;e335232f7491:39717 {}] regionserver.HRegionServer(1321): Waiting on 1 regions to close 2024-11-24T16:45:18,459 DEBUG [RS:1;e335232f7491:39717 {}] regionserver.HRegionServer(1325): Online Regions={8d688d1c7c98ba065aee654c8fa6e77a=testBlockEvictionOnRegionMove1732466716044,,1732466716067.8d688d1c7c98ba065aee654c8fa6e77a.} 2024-11-24T16:45:18,459 DEBUG [RS_CLOSE_REGION-regionserver/e335232f7491:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 8d688d1c7c98ba065aee654c8fa6e77a, disabling compactions & flushes 2024-11-24T16:45:18,459 INFO [RS:0;e335232f7491:45465 {}] regionserver.HRegionServer(1321): Waiting on 1 regions to close 2024-11-24T16:45:18,459 INFO [RS_CLOSE_REGION-regionserver/e335232f7491:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testBlockEvictionOnRegionMove1732466716044,,1732466716067.8d688d1c7c98ba065aee654c8fa6e77a. 2024-11-24T16:45:18,459 DEBUG [RS:0;e335232f7491:45465 {}] regionserver.HRegionServer(1325): Online Regions={1588230740=hbase:meta,,1.1588230740} 2024-11-24T16:45:18,459 DEBUG [RS_CLOSE_REGION-regionserver/e335232f7491:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testBlockEvictionOnRegionMove1732466716044,,1732466716067.8d688d1c7c98ba065aee654c8fa6e77a. 2024-11-24T16:45:18,459 DEBUG [RS:1;e335232f7491:39717 {}] regionserver.HRegionServer(1351): Waiting on 8d688d1c7c98ba065aee654c8fa6e77a 2024-11-24T16:45:18,459 DEBUG [RS_CLOSE_REGION-regionserver/e335232f7491:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testBlockEvictionOnRegionMove1732466716044,,1732466716067.8d688d1c7c98ba065aee654c8fa6e77a. after waiting 0 ms 2024-11-24T16:45:18,459 DEBUG [RS:0;e335232f7491:45465 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-24T16:45:18,459 INFO [RS_CLOSE_META-regionserver/e335232f7491:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1662): Closing region hbase:meta,,1.1588230740 during a graceful stop, and cache persistence is on, so setting evict on close to false. 2024-11-24T16:45:18,459 DEBUG [RS_CLOSE_REGION-regionserver/e335232f7491:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testBlockEvictionOnRegionMove1732466716044,,1732466716067.8d688d1c7c98ba065aee654c8fa6e77a. 2024-11-24T16:45:18,459 DEBUG [RS_CLOSE_META-regionserver/e335232f7491:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-24T16:45:18,459 INFO [RS_CLOSE_META-regionserver/e335232f7491:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-24T16:45:18,459 DEBUG [RS_CLOSE_META-regionserver/e335232f7491:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-24T16:45:18,459 DEBUG [RS_CLOSE_META-regionserver/e335232f7491:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-24T16:45:18,459 DEBUG [RS_CLOSE_META-regionserver/e335232f7491:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-24T16:45:18,460 DEBUG [StoreFileCloser-8d688d1c7c98ba065aee654c8fa6e77a-family-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9797cf4c-7cf0-a873-8ba7-f30267b4a2f7/39f4fdd4-f188-1d18-4286-5b0c4dc27cad/data/default/testBlockEvictionOnRegionMove1732466716044/8d688d1c7c98ba065aee654c8fa6e77a/family/b4ec7c78b33d4365b798fb9a751d2936 2024-11-24T16:45:18,460 INFO [RS_CLOSE_META-regionserver/e335232f7491:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=3.70 KB heapSize=6.72 KB 2024-11-24T16:45:18,465 DEBUG [RS_CLOSE_REGION-regionserver/e335232f7491:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9797cf4c-7cf0-a873-8ba7-f30267b4a2f7/39f4fdd4-f188-1d18-4286-5b0c4dc27cad/data/default/testBlockEvictionOnRegionMove1732466716044/8d688d1c7c98ba065aee654c8fa6e77a/recovered.edits/12.seqid, newMaxSeqId=12, maxSeqId=9 2024-11-24T16:45:18,465 INFO [RS_CLOSE_REGION-regionserver/e335232f7491:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testBlockEvictionOnRegionMove1732466716044,,1732466716067.8d688d1c7c98ba065aee654c8fa6e77a. 2024-11-24T16:45:18,465 DEBUG [RS_CLOSE_REGION-regionserver/e335232f7491:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 8d688d1c7c98ba065aee654c8fa6e77a: Waiting for close lock at 1732466718458Running coprocessor pre-close hooks at 1732466718459 (+1 ms)Disabling compacts and flushes for region at 1732466718459Disabling writes for close at 1732466718459Writing region close event to WAL at 1732466718460 (+1 ms)Running coprocessor post-close hooks at 1732466718465 (+5 ms)Closed at 1732466718465 2024-11-24T16:45:18,465 DEBUG [RS_CLOSE_REGION-regionserver/e335232f7491:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testBlockEvictionOnRegionMove1732466716044,,1732466716067.8d688d1c7c98ba065aee654c8fa6e77a. 2024-11-24T16:45:18,471 DEBUG [RS_CLOSE_META-regionserver/e335232f7491:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 5da3326533994422bab81cebfe00893d 2024-11-24T16:45:18,484 INFO [regionserver/e335232f7491:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-24T16:45:18,490 DEBUG [RS_CLOSE_META-regionserver/e335232f7491:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9797cf4c-7cf0-a873-8ba7-f30267b4a2f7/39f4fdd4-f188-1d18-4286-5b0c4dc27cad/data/hbase/meta/1588230740/.tmp/info/5da3326533994422bab81cebfe00893d is 205, key is testBlockEvictionOnRegionMove1732466716044,,1732466716067.8d688d1c7c98ba065aee654c8fa6e77a./info:regioninfo/1732466718390/Put/seqid=0 2024-11-24T16:45:18,490 INFO [RS_CLOSE_META-regionserver/e335232f7491:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=3.48 KB at sequenceid=15 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9797cf4c-7cf0-a873-8ba7-f30267b4a2f7/39f4fdd4-f188-1d18-4286-5b0c4dc27cad/data/hbase/meta/1588230740/.tmp/info/5da3326533994422bab81cebfe00893d 2024-11-24T16:45:18,501 DEBUG [RS_CLOSE_META-regionserver/e335232f7491:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 0c2be411276d46a785be351eed27d378 2024-11-24T16:45:18,512 INFO [regionserver/e335232f7491:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-24T16:45:18,515 DEBUG [RS_CLOSE_META-regionserver/e335232f7491:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9797cf4c-7cf0-a873-8ba7-f30267b4a2f7/39f4fdd4-f188-1d18-4286-5b0c4dc27cad/data/hbase/meta/1588230740/.tmp/ns/0c2be411276d46a785be351eed27d378 is 43, key is default/ns:d/1732466715890/Put/seqid=0 2024-11-24T16:45:18,516 INFO [RS_CLOSE_META-regionserver/e335232f7491:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=74 B at sequenceid=15 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9797cf4c-7cf0-a873-8ba7-f30267b4a2f7/39f4fdd4-f188-1d18-4286-5b0c4dc27cad/data/hbase/meta/1588230740/.tmp/ns/0c2be411276d46a785be351eed27d378 2024-11-24T16:45:18,526 DEBUG [RS_CLOSE_META-regionserver/e335232f7491:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFile$WriterFactory(308): Unable to set drop behind on f631812b51dd4749a8d718d638295a9b 2024-11-24T16:45:18,540 DEBUG [RS_CLOSE_META-regionserver/e335232f7491:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9797cf4c-7cf0-a873-8ba7-f30267b4a2f7/39f4fdd4-f188-1d18-4286-5b0c4dc27cad/data/hbase/meta/1588230740/.tmp/table/f631812b51dd4749a8d718d638295a9b is 78, key is testBlockEvictionOnRegionMove1732466716044/table:state/1732466716476/Put/seqid=0 2024-11-24T16:45:18,541 INFO [RS_CLOSE_META-regionserver/e335232f7491:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=148 B at sequenceid=15 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9797cf4c-7cf0-a873-8ba7-f30267b4a2f7/39f4fdd4-f188-1d18-4286-5b0c4dc27cad/data/hbase/meta/1588230740/.tmp/table/f631812b51dd4749a8d718d638295a9b 2024-11-24T16:45:18,543 DEBUG [RS_CLOSE_META-regionserver/e335232f7491:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9797cf4c-7cf0-a873-8ba7-f30267b4a2f7/39f4fdd4-f188-1d18-4286-5b0c4dc27cad/data/hbase/meta/1588230740/.tmp/info/5da3326533994422bab81cebfe00893d as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9797cf4c-7cf0-a873-8ba7-f30267b4a2f7/39f4fdd4-f188-1d18-4286-5b0c4dc27cad/data/hbase/meta/1588230740/info/5da3326533994422bab81cebfe00893d 2024-11-24T16:45:18,545 DEBUG [RS_CLOSE_META-regionserver/e335232f7491:0-0 {event_type=M_RS_CLOSE_META}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9797cf4c-7cf0-a873-8ba7-f30267b4a2f7/39f4fdd4-f188-1d18-4286-5b0c4dc27cad/data/hbase/meta/1588230740/info/5da3326533994422bab81cebfe00893d, delay=985 ms 2024-11-24T16:45:18,546 INFO [RS_CLOSE_META-regionserver/e335232f7491:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9797cf4c-7cf0-a873-8ba7-f30267b4a2f7/39f4fdd4-f188-1d18-4286-5b0c4dc27cad/data/hbase/meta/1588230740/info/5da3326533994422bab81cebfe00893d, entries=15, sequenceid=15, filesize=7.6 K 2024-11-24T16:45:18,546 DEBUG [RS_CLOSE_META-regionserver/e335232f7491:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9797cf4c-7cf0-a873-8ba7-f30267b4a2f7/39f4fdd4-f188-1d18-4286-5b0c4dc27cad/data/hbase/meta/1588230740/.tmp/ns/0c2be411276d46a785be351eed27d378 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9797cf4c-7cf0-a873-8ba7-f30267b4a2f7/39f4fdd4-f188-1d18-4286-5b0c4dc27cad/data/hbase/meta/1588230740/ns/0c2be411276d46a785be351eed27d378 2024-11-24T16:45:18,547 DEBUG [RS_CLOSE_META-regionserver/e335232f7491:0-0 {event_type=M_RS_CLOSE_META}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9797cf4c-7cf0-a873-8ba7-f30267b4a2f7/39f4fdd4-f188-1d18-4286-5b0c4dc27cad/data/hbase/meta/1588230740/ns/0c2be411276d46a785be351eed27d378, delay=963 ms 2024-11-24T16:45:18,549 INFO [RS_CLOSE_META-regionserver/e335232f7491:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9797cf4c-7cf0-a873-8ba7-f30267b4a2f7/39f4fdd4-f188-1d18-4286-5b0c4dc27cad/data/hbase/meta/1588230740/ns/0c2be411276d46a785be351eed27d378, entries=2, sequenceid=15, filesize=5.0 K 2024-11-24T16:45:18,549 DEBUG [RS_CLOSE_META-regionserver/e335232f7491:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9797cf4c-7cf0-a873-8ba7-f30267b4a2f7/39f4fdd4-f188-1d18-4286-5b0c4dc27cad/data/hbase/meta/1588230740/.tmp/table/f631812b51dd4749a8d718d638295a9b as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9797cf4c-7cf0-a873-8ba7-f30267b4a2f7/39f4fdd4-f188-1d18-4286-5b0c4dc27cad/data/hbase/meta/1588230740/table/f631812b51dd4749a8d718d638295a9b 2024-11-24T16:45:18,550 DEBUG [RS_CLOSE_META-regionserver/e335232f7491:0-0 {event_type=M_RS_CLOSE_META}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9797cf4c-7cf0-a873-8ba7-f30267b4a2f7/39f4fdd4-f188-1d18-4286-5b0c4dc27cad/data/hbase/meta/1588230740/table/f631812b51dd4749a8d718d638295a9b, delay=988 ms 2024-11-24T16:45:18,551 INFO [RS_CLOSE_META-regionserver/e335232f7491:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9797cf4c-7cf0-a873-8ba7-f30267b4a2f7/39f4fdd4-f188-1d18-4286-5b0c4dc27cad/data/hbase/meta/1588230740/table/f631812b51dd4749a8d718d638295a9b, entries=2, sequenceid=15, filesize=5.3 K 2024-11-24T16:45:18,552 INFO [RS_CLOSE_META-regionserver/e335232f7491:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~3.70 KB/3789, heapSize ~6.42 KB/6576, currentSize=0 B/0 for 1588230740 in 93ms, sequenceid=15, compaction requested=false 2024-11-24T16:45:18,552 DEBUG [RS_CLOSE_META-regionserver/e335232f7491:0-0 {event_type=M_RS_CLOSE_META}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta' 2024-11-24T16:45:18,553 DEBUG [StoreFileCloser-1588230740-info-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9797cf4c-7cf0-a873-8ba7-f30267b4a2f7/39f4fdd4-f188-1d18-4286-5b0c4dc27cad/data/hbase/meta/1588230740/info/5da3326533994422bab81cebfe00893d 2024-11-24T16:45:18,554 DEBUG [StoreFileCloser-1588230740-ns-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9797cf4c-7cf0-a873-8ba7-f30267b4a2f7/39f4fdd4-f188-1d18-4286-5b0c4dc27cad/data/hbase/meta/1588230740/ns/0c2be411276d46a785be351eed27d378 2024-11-24T16:45:18,555 DEBUG [StoreFileCloser-1588230740-table-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9797cf4c-7cf0-a873-8ba7-f30267b4a2f7/39f4fdd4-f188-1d18-4286-5b0c4dc27cad/data/hbase/meta/1588230740/table/f631812b51dd4749a8d718d638295a9b 2024-11-24T16:45:18,561 DEBUG [RS_CLOSE_META-regionserver/e335232f7491:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9797cf4c-7cf0-a873-8ba7-f30267b4a2f7/39f4fdd4-f188-1d18-4286-5b0c4dc27cad/data/hbase/meta/1588230740/recovered.edits/18.seqid, newMaxSeqId=18, maxSeqId=1 2024-11-24T16:45:18,561 DEBUG [RS_CLOSE_META-regionserver/e335232f7491:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-24T16:45:18,562 INFO [RS_CLOSE_META-regionserver/e335232f7491:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-24T16:45:18,562 DEBUG [RS_CLOSE_META-regionserver/e335232f7491:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1732466718459Running coprocessor pre-close hooks at 1732466718459Disabling compacts and flushes for region at 1732466718459Disabling writes for close at 1732466718459Obtaining lock to block concurrent updates at 1732466718460 (+1 ms)Preparing flush snapshotting stores in 1588230740 at 1732466718460Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=3789, getHeapSize=6816, getOffHeapSize=0, getCellsCount=27 at 1732466718460Flushing stores of hbase:meta,,1.1588230740 at 1732466718461 (+1 ms)Flushing 1588230740/info: creating writer at 1732466718461Flushing 1588230740/info: appending metadata at 1732466718489 (+28 ms)Flushing 1588230740/info: closing flushed file at 1732466718489Flushing 1588230740/ns: creating writer at 1732466718493 (+4 ms)Flushing 1588230740/ns: appending metadata at 1732466718514 (+21 ms)Flushing 1588230740/ns: closing flushed file at 1732466718514Flushing 1588230740/table: creating writer at 1732466718518 (+4 ms)Flushing 1588230740/table: appending metadata at 1732466718539 (+21 ms)Flushing 1588230740/table: closing flushed file at 1732466718539Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@57a51bc0: reopening flushed file at 1732466718543 (+4 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@2023bf49: reopening flushed file at 1732466718546 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5a329e72: reopening flushed file at 1732466718549 (+3 ms)Finished flush of dataSize ~3.70 KB/3789, heapSize ~6.42 KB/6576, currentSize=0 B/0 for 1588230740 in 93ms, sequenceid=15, compaction requested=false at 1732466718552 (+3 ms)Writing region close event to WAL at 1732466718555 (+3 ms)Running coprocessor post-close hooks at 1732466718561 (+6 ms)Closed at 1732466718562 (+1 ms) 2024-11-24T16:45:18,562 DEBUG [RS_CLOSE_META-regionserver/e335232f7491:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-11-24T16:45:18,659 INFO [RS:1;e335232f7491:39717 {}] regionserver.HRegionServer(976): stopping server e335232f7491,39717,1732466714030; all regions closed. 2024-11-24T16:45:18,659 INFO [RS:0;e335232f7491:45465 {}] regionserver.HRegionServer(976): stopping server e335232f7491,45465,1732466713886; all regions closed. 2024-11-24T16:45:18,664 DEBUG [RS:1;e335232f7491:39717 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9797cf4c-7cf0-a873-8ba7-f30267b4a2f7/39f4fdd4-f188-1d18-4286-5b0c4dc27cad/oldWALs 2024-11-24T16:45:18,664 DEBUG [RS:0;e335232f7491:45465 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9797cf4c-7cf0-a873-8ba7-f30267b4a2f7/39f4fdd4-f188-1d18-4286-5b0c4dc27cad/oldWALs 2024-11-24T16:45:18,665 INFO [RS:0;e335232f7491:45465 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL e335232f7491%2C45465%2C1732466713886.meta:.meta(num 1732466715753) 2024-11-24T16:45:18,665 INFO [RS:1;e335232f7491:39717 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL e335232f7491%2C39717%2C1732466714030:(num 1732466715324) 2024-11-24T16:45:18,665 DEBUG [RS:1;e335232f7491:39717 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T16:45:18,665 INFO [RS:1;e335232f7491:39717 {}] regionserver.LeaseManager(133): Closed leases 2024-11-24T16:45:18,665 INFO [RS:1;e335232f7491:39717 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-24T16:45:18,665 INFO [RS:1;e335232f7491:39717 {}] hbase.ChoreService(370): Chore service for: regionserver/e335232f7491:0 had [ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS] on shutdown 2024-11-24T16:45:18,665 INFO [RS:1;e335232f7491:39717 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-24T16:45:18,666 INFO [RS:1;e335232f7491:39717 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-24T16:45:18,666 INFO [regionserver/e335232f7491:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-24T16:45:18,666 INFO [RS:1;e335232f7491:39717 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-24T16:45:18,666 INFO [RS:1;e335232f7491:39717 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-24T16:45:18,666 INFO [RS:1;e335232f7491:39717 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:39717 2024-11-24T16:45:18,666 DEBUG [RS:0;e335232f7491:45465 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9797cf4c-7cf0-a873-8ba7-f30267b4a2f7/39f4fdd4-f188-1d18-4286-5b0c4dc27cad/oldWALs 2024-11-24T16:45:18,666 INFO [RS:0;e335232f7491:45465 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL e335232f7491%2C45465%2C1732466713886:(num 1732466715275) 2024-11-24T16:45:18,667 DEBUG [RS:0;e335232f7491:45465 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T16:45:18,667 INFO [RS:0;e335232f7491:45465 {}] regionserver.LeaseManager(133): Closed leases 2024-11-24T16:45:18,667 INFO [RS:0;e335232f7491:45465 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-24T16:45:18,667 INFO [RS:0;e335232f7491:45465 {}] hbase.ChoreService(370): Chore service for: regionserver/e335232f7491:0 had [ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-11-24T16:45:18,667 INFO [RS:0;e335232f7491:45465 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-24T16:45:18,667 INFO [regionserver/e335232f7491:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-24T16:45:18,667 INFO [RS:0;e335232f7491:45465 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:45465 2024-11-24T16:45:18,671 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39717-0x1014ade0b8c0002, quorum=127.0.0.1:60963, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/e335232f7491,39717,1732466714030 2024-11-24T16:45:18,672 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45629-0x1014ade0b8c0000, quorum=127.0.0.1:60963, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-24T16:45:18,672 INFO [RS:1;e335232f7491:39717 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-24T16:45:18,672 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45465-0x1014ade0b8c0001, quorum=127.0.0.1:60963, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/e335232f7491,45465,1732466713886 2024-11-24T16:45:18,672 INFO [RS:0;e335232f7491:45465 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-24T16:45:18,674 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [e335232f7491,39717,1732466714030] 2024-11-24T16:45:18,676 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/e335232f7491,39717,1732466714030 already deleted, retry=false 2024-11-24T16:45:18,676 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; e335232f7491,39717,1732466714030 expired; onlineServers=1 2024-11-24T16:45:18,676 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [e335232f7491,45465,1732466713886] 2024-11-24T16:45:18,678 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/e335232f7491,45465,1732466713886 already deleted, retry=false 2024-11-24T16:45:18,678 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; e335232f7491,45465,1732466713886 expired; onlineServers=0 2024-11-24T16:45:18,679 INFO [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master 'e335232f7491,45629,1732466713116' ***** 2024-11-24T16:45:18,679 INFO [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0 2024-11-24T16:45:18,679 INFO [M:0;e335232f7491:45629 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-24T16:45:18,679 INFO [M:0;e335232f7491:45629 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-24T16:45:18,679 DEBUG [M:0;e335232f7491:45629 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2024-11-24T16:45:18,679 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2024-11-24T16:45:18,679 DEBUG [M:0;e335232f7491:45629 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2024-11-24T16:45:18,679 DEBUG [master/e335232f7491:0:becomeActiveMaster-HFileCleaner.small.0-1732466715008 {}] cleaner.HFileCleaner(306): Exit Thread[master/e335232f7491:0:becomeActiveMaster-HFileCleaner.small.0-1732466715008,5,FailOnTimeoutGroup] 2024-11-24T16:45:18,679 DEBUG [master/e335232f7491:0:becomeActiveMaster-HFileCleaner.large.0-1732466715007 {}] cleaner.HFileCleaner(306): Exit Thread[master/e335232f7491:0:becomeActiveMaster-HFileCleaner.large.0-1732466715007,5,FailOnTimeoutGroup] 2024-11-24T16:45:18,679 INFO [M:0;e335232f7491:45629 {}] hbase.ChoreService(370): Chore service for: master/e335232f7491:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown 2024-11-24T16:45:18,679 INFO [M:0;e335232f7491:45629 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-24T16:45:18,679 DEBUG [M:0;e335232f7491:45629 {}] master.HMaster(1795): Stopping service threads 2024-11-24T16:45:18,680 INFO [M:0;e335232f7491:45629 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2024-11-24T16:45:18,680 INFO [M:0;e335232f7491:45629 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-24T16:45:18,680 INFO [M:0;e335232f7491:45629 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-24T16:45:18,680 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2024-11-24T16:45:18,681 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45629-0x1014ade0b8c0000, quorum=127.0.0.1:60963, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-11-24T16:45:18,681 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45629-0x1014ade0b8c0000, quorum=127.0.0.1:60963, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T16:45:18,681 DEBUG [M:0;e335232f7491:45629 {}] zookeeper.ZKUtil(347): master:45629-0x1014ade0b8c0000, quorum=127.0.0.1:60963, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error) 2024-11-24T16:45:18,681 WARN [M:0;e335232f7491:45629 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null 2024-11-24T16:45:18,681 INFO [M:0;e335232f7491:45629 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9797cf4c-7cf0-a873-8ba7-f30267b4a2f7/39f4fdd4-f188-1d18-4286-5b0c4dc27cad/.lastflushedseqids 2024-11-24T16:45:18,690 INFO [M:0;e335232f7491:45629 {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-24T16:45:18,690 INFO [M:0;e335232f7491:45629 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2024-11-24T16:45:18,691 DEBUG [M:0;e335232f7491:45629 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-24T16:45:18,691 INFO [M:0;e335232f7491:45629 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-24T16:45:18,691 DEBUG [M:0;e335232f7491:45629 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-24T16:45:18,691 DEBUG [M:0;e335232f7491:45629 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-24T16:45:18,691 DEBUG [M:0;e335232f7491:45629 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-24T16:45:18,691 INFO [M:0;e335232f7491:45629 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=34.67 KB heapSize=43.02 KB 2024-11-24T16:45:18,702 DEBUG [M:0;e335232f7491:45629 {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 4484f3c8b51843ee869c21548dcb263e 2024-11-24T16:45:18,715 DEBUG [M:0;e335232f7491:45629 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9797cf4c-7cf0-a873-8ba7-f30267b4a2f7/39f4fdd4-f188-1d18-4286-5b0c4dc27cad/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/4484f3c8b51843ee869c21548dcb263e is 82, key is hbase:meta,,1/info:regioninfo/1732466715826/Put/seqid=0 2024-11-24T16:45:18,716 INFO [M:0;e335232f7491:45629 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=504 B at sequenceid=84 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9797cf4c-7cf0-a873-8ba7-f30267b4a2f7/39f4fdd4-f188-1d18-4286-5b0c4dc27cad/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/4484f3c8b51843ee869c21548dcb263e 2024-11-24T16:45:18,724 DEBUG [M:0;e335232f7491:45629 {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on b19486ed34724858b9258862f34bc64c 2024-11-24T16:45:18,737 DEBUG [M:0;e335232f7491:45629 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9797cf4c-7cf0-a873-8ba7-f30267b4a2f7/39f4fdd4-f188-1d18-4286-5b0c4dc27cad/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/b19486ed34724858b9258862f34bc64c is 778, key is \x00\x00\x00\x00\x00\x00\x00\x04/proc:d/1732466716481/Put/seqid=0 2024-11-24T16:45:18,738 INFO [M:0;e335232f7491:45629 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=34.05 KB at sequenceid=84 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9797cf4c-7cf0-a873-8ba7-f30267b4a2f7/39f4fdd4-f188-1d18-4286-5b0c4dc27cad/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/b19486ed34724858b9258862f34bc64c 2024-11-24T16:45:18,746 DEBUG [M:0;e335232f7491:45629 {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 1663da6105ab4e708e35eb4c7d65e8f9 2024-11-24T16:45:18,758 DEBUG [M:0;e335232f7491:45629 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9797cf4c-7cf0-a873-8ba7-f30267b4a2f7/39f4fdd4-f188-1d18-4286-5b0c4dc27cad/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/1663da6105ab4e708e35eb4c7d65e8f9 is 69, key is e335232f7491,39717,1732466714030/rs:state/1732466715153/Put/seqid=0 2024-11-24T16:45:18,759 INFO [M:0;e335232f7491:45629 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=130 B at sequenceid=84 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9797cf4c-7cf0-a873-8ba7-f30267b4a2f7/39f4fdd4-f188-1d18-4286-5b0c4dc27cad/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/1663da6105ab4e708e35eb4c7d65e8f9 2024-11-24T16:45:18,761 DEBUG [M:0;e335232f7491:45629 {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9797cf4c-7cf0-a873-8ba7-f30267b4a2f7/39f4fdd4-f188-1d18-4286-5b0c4dc27cad/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/4484f3c8b51843ee869c21548dcb263e as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9797cf4c-7cf0-a873-8ba7-f30267b4a2f7/39f4fdd4-f188-1d18-4286-5b0c4dc27cad/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/4484f3c8b51843ee869c21548dcb263e 2024-11-24T16:45:18,763 INFO [M:0;e335232f7491:45629 {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9797cf4c-7cf0-a873-8ba7-f30267b4a2f7/39f4fdd4-f188-1d18-4286-5b0c4dc27cad/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/4484f3c8b51843ee869c21548dcb263e, entries=8, sequenceid=84, filesize=5.5 K 2024-11-24T16:45:18,763 DEBUG [M:0;e335232f7491:45629 {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9797cf4c-7cf0-a873-8ba7-f30267b4a2f7/39f4fdd4-f188-1d18-4286-5b0c4dc27cad/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/b19486ed34724858b9258862f34bc64c as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9797cf4c-7cf0-a873-8ba7-f30267b4a2f7/39f4fdd4-f188-1d18-4286-5b0c4dc27cad/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/b19486ed34724858b9258862f34bc64c 2024-11-24T16:45:18,765 INFO [M:0;e335232f7491:45629 {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9797cf4c-7cf0-a873-8ba7-f30267b4a2f7/39f4fdd4-f188-1d18-4286-5b0c4dc27cad/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/b19486ed34724858b9258862f34bc64c, entries=9, sequenceid=84, filesize=6.5 K 2024-11-24T16:45:18,765 DEBUG [M:0;e335232f7491:45629 {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9797cf4c-7cf0-a873-8ba7-f30267b4a2f7/39f4fdd4-f188-1d18-4286-5b0c4dc27cad/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/1663da6105ab4e708e35eb4c7d65e8f9 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9797cf4c-7cf0-a873-8ba7-f30267b4a2f7/39f4fdd4-f188-1d18-4286-5b0c4dc27cad/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/1663da6105ab4e708e35eb4c7d65e8f9 2024-11-24T16:45:18,767 INFO [M:0;e335232f7491:45629 {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9797cf4c-7cf0-a873-8ba7-f30267b4a2f7/39f4fdd4-f188-1d18-4286-5b0c4dc27cad/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/1663da6105ab4e708e35eb4c7d65e8f9, entries=2, sequenceid=84, filesize=5.1 K 2024-11-24T16:45:18,767 INFO [M:0;e335232f7491:45629 {}] regionserver.HRegion(3140): Finished flush of dataSize ~34.67 KB/35506, heapSize ~42.73 KB/43752, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 76ms, sequenceid=84, compaction requested=false 2024-11-24T16:45:18,770 INFO [M:0;e335232f7491:45629 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-24T16:45:18,770 DEBUG [M:0;e335232f7491:45629 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1732466718691Disabling compacts and flushes for region at 1732466718691Disabling writes for close at 1732466718691Obtaining lock to block concurrent updates at 1732466718691Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1732466718691Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=35506, getHeapSize=43992, getOffHeapSize=0, getCellsCount=99 at 1732466718691Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1732466718692 (+1 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1732466718692Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1732466718715 (+23 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1732466718715Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1732466718718 (+3 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1732466718737 (+19 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1732466718737Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1732466718740 (+3 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1732466718758 (+18 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1732466718758Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@67f69d6a: reopening flushed file at 1732466718761 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@78a55e7f: reopening flushed file at 1732466718763 (+2 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@653bb434: reopening flushed file at 1732466718765 (+2 ms)Finished flush of dataSize ~34.67 KB/35506, heapSize ~42.73 KB/43752, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 76ms, sequenceid=84, compaction requested=false at 1732466718767 (+2 ms)Writing region close event to WAL at 1732466718770 (+3 ms)Closed at 1732466718770 2024-11-24T16:45:18,772 INFO [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-24T16:45:18,772 INFO [M:0;e335232f7491:45629 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down. 2024-11-24T16:45:18,773 INFO [M:0;e335232f7491:45629 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:45629 2024-11-24T16:45:18,773 INFO [M:0;e335232f7491:45629 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-24T16:45:18,775 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39717-0x1014ade0b8c0002, quorum=127.0.0.1:60963, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-24T16:45:18,775 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45465-0x1014ade0b8c0001, quorum=127.0.0.1:60963, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-24T16:45:18,775 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39717-0x1014ade0b8c0002, quorum=127.0.0.1:60963, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-24T16:45:18,775 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45465-0x1014ade0b8c0001, quorum=127.0.0.1:60963, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-24T16:45:18,775 INFO [RS:1;e335232f7491:39717 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-24T16:45:18,775 INFO [RS:0;e335232f7491:45465 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-24T16:45:18,775 INFO [RS:1;e335232f7491:39717 {}] regionserver.HRegionServer(1031): Exiting; stopping=e335232f7491,39717,1732466714030; zookeeper connection closed. 2024-11-24T16:45:18,775 INFO [RS:0;e335232f7491:45465 {}] regionserver.HRegionServer(1031): Exiting; stopping=e335232f7491,45465,1732466713886; zookeeper connection closed. 2024-11-24T16:45:18,776 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@1f21f836 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@1f21f836 2024-11-24T16:45:18,776 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@7e28898c {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@7e28898c 2024-11-24T16:45:18,776 INFO [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 2 regionserver(s) complete 2024-11-24T16:45:18,875 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45629-0x1014ade0b8c0000, quorum=127.0.0.1:60963, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-24T16:45:18,875 INFO [M:0;e335232f7491:45629 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-24T16:45:18,876 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45629-0x1014ade0b8c0000, quorum=127.0.0.1:60963, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-24T16:45:18,880 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers 2024-11-24T16:45:18,899 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down 2024-11-24T16:45:18,900 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-24T16:45:18,908 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: io.hfile.TestBlockEvictionOnRegionMovement#testBlockEvictionOnRegionMove Thread=49 (was 12) Potentially hanging thread: ForkJoinPool.commonPool-worker-3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.ForkJoinPool.awaitWork(ForkJoinPool.java:1724) java.base@17.0.11/java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1623) java.base@17.0.11/java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:165) Potentially hanging thread: Time-limited test.named-queue-events-pool-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) app//com.lmax.disruptor.BlockingWaitStrategy.waitFor(BlockingWaitStrategy.java:47) app//com.lmax.disruptor.ProcessingSequenceBarrier.waitFor(ProcessingSequenceBarrier.java:56) app//com.lmax.disruptor.BatchEventProcessor.processEvents(BatchEventProcessor.java:159) app//com.lmax.disruptor.BatchEventProcessor.run(BatchEventProcessor.java:125) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: SessionTracker java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.zookeeper.server.SessionTrackerImpl.run(SessionTrackerImpl.java:163) Potentially hanging thread: region-location-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1170) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: master/e335232f7491:0:becomeActiveMaster-MemStoreChunkPool Statistics java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RpcClient-timer-pool-0 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.waitForNextTick(HashedWheelTimer.java:598) app//org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:494) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-3-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Async-Client-Retry-Timer-pool-0 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.waitForNextTick(HashedWheelTimer.java:598) app//org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:494) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: hfile-prefetch-1732466718378 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1177) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ForkJoinPool.commonPool-worker-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkUntil(LockSupport.java:410) java.base@17.0.11/java.util.concurrent.ForkJoinPool.awaitWork(ForkJoinPool.java:1726) java.base@17.0.11/java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1623) java.base@17.0.11/java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:165) Potentially hanging thread: org.apache.hadoop.fs.FileSystem$Statistics$StatisticsDataReferenceCleaner java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:155) java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:176) app//org.apache.hadoop.fs.FileSystem$Statistics$StatisticsDataReferenceCleaner.run(FileSystem.java:4171) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: regionserver/e335232f7491:0.procedureResultReporter java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.RemoteProcedureResultReporter.run(RemoteProcedureResultReporter.java:75) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-4-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: hfile-prefetch-1732466718545 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1177) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HMaster-EventLoopGroup-1-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ForkJoinPool.commonPool-worker-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.ForkJoinPool.awaitWork(ForkJoinPool.java:1724) java.base@17.0.11/java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1623) java.base@17.0.11/java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:165) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-4-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-3-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HMaster-EventLoopGroup-1-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HMaster-EventLoopGroup-1-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-3-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: hfile-prefetch-1732466716852 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Idle-Rpc-Conn-Sweeper-pool-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Timer for 'HBase' metrics system java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.util.TimerThread.mainLoop(Timer.java:563) java.base@17.0.11/java.util.TimerThread.run(Timer.java:516) Potentially hanging thread: Monitor thread for TaskMonitor java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hbase.monitoring.TaskMonitor$MonitorRunnable.run(TaskMonitor.java:325) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: weak-ref-cleaner-strictcontextstorage java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:155) java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:176) app//io.opentelemetry.context.StrictContextStorage$PendingScopes.run(StrictContextStorage.java:269) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: regionserver/e335232f7491:0.procedureResultReporter java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.RemoteProcedureResultReporter.run(RemoteProcedureResultReporter.java:75) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-4-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: hfile-prefetch-1732466718548 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1177) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: master/e335232f7491:0:becomeActiveMaster-MemStoreChunkPool Statistics java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: SnapshotHandlerChoreCleaner java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-4 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HBase-Metrics2-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=346 (was 287) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=656 (was 714), ProcessCount=11 (was 11), AvailableMemoryMB=6607 (was 6951) 2024-11-24T16:45:18,914 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: io.hfile.TestBlockEvictionOnRegionMovement#testBlockEvictionOnGracefulStop Thread=49, OpenFileDescriptor=346, MaxFileDescriptor=1048576, SystemLoadAverage=656, ProcessCount=11, AvailableMemoryMB=6607 2024-11-24T16:45:18,914 INFO [Time-limited test {}] hbase.HBaseTestingUtil(401): System.getProperty("hadoop.log.dir") already set to: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9797cf4c-7cf0-a873-8ba7-f30267b4a2f7/hadoop-log-dir so I do NOT create it in target/test-data/98608d84-f0c0-ab20-75c0-76a28c1904b5 2024-11-24T16:45:18,914 INFO [Time-limited test {}] hbase.HBaseTestingUtil(401): System.getProperty("hadoop.tmp.dir") already set to: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9797cf4c-7cf0-a873-8ba7-f30267b4a2f7/hadoop-tmp-dir so I do NOT create it in target/test-data/98608d84-f0c0-ab20-75c0-76a28c1904b5 2024-11-24T16:45:18,914 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-24T16:45:18,917 INFO [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/98608d84-f0c0-ab20-75c0-76a28c1904b5/cluster_f501b2b8-030e-af9d-c1b1-b1033a148d78, deleteOnExit=true 2024-11-24T16:45:18,920 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/98608d84-f0c0-ab20-75c0-76a28c1904b5/cluster_f501b2b8-030e-af9d-c1b1-b1033a148d78/zookeeper_0, clientPort=63542, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/98608d84-f0c0-ab20-75c0-76a28c1904b5/cluster_f501b2b8-030e-af9d-c1b1-b1033a148d78/zookeeper_0/version-2, dataDirSize=457 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/98608d84-f0c0-ab20-75c0-76a28c1904b5/cluster_f501b2b8-030e-af9d-c1b1-b1033a148d78/zookeeper_0/version-2, dataLogSize=457 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0 2024-11-24T16:45:18,921 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=63542 2024-11-24T16:45:18,933 INFO [Time-limited test {}] util.FSUtils(489): Created version file at /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9797cf4c-7cf0-a873-8ba7-f30267b4a2f7/39f4fdd4-f188-1d18-4286-5b0c4dc27cad with version=8 2024-11-24T16:45:18,933 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1141): The hbase.fs.tmp.dir is set to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9797cf4c-7cf0-a873-8ba7-f30267b4a2f7/39f4fdd4-f188-1d18-4286-5b0c4dc27cad/hbase-staging 2024-11-24T16:45:18,936 INFO [Time-limited test {}] client.ConnectionUtils(128): master/e335232f7491:0 server-side Connection retries=45 2024-11-24T16:45:18,936 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-24T16:45:18,936 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-24T16:45:18,937 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-24T16:45:18,937 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-24T16:45:18,937 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-24T16:45:18,937 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService 2024-11-24T16:45:18,937 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-24T16:45:18,938 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:36357 2024-11-24T16:45:18,939 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:36357 connecting to ZooKeeper ensemble=127.0.0.1:63542 2024-11-24T16:45:18,947 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:363570x0, quorum=127.0.0.1:63542, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-24T16:45:18,948 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:36357-0x1014ade23570000 connected 2024-11-24T16:45:18,963 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-24T16:45:18,964 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-24T16:45:18,965 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:36357-0x1014ade23570000, quorum=127.0.0.1:63542, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-24T16:45:18,965 INFO [Time-limited test {}] master.HMaster(525): hbase.rootdir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9797cf4c-7cf0-a873-8ba7-f30267b4a2f7/39f4fdd4-f188-1d18-4286-5b0c4dc27cad, hbase.cluster.distributed=false 2024-11-24T16:45:18,967 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:36357-0x1014ade23570000, quorum=127.0.0.1:63542, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-24T16:45:18,967 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=36357 2024-11-24T16:45:18,968 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=36357 2024-11-24T16:45:18,968 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=36357 2024-11-24T16:45:18,968 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=36357 2024-11-24T16:45:18,968 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=36357 2024-11-24T16:45:18,985 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/e335232f7491:0 server-side Connection retries=45 2024-11-24T16:45:18,985 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-24T16:45:18,985 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-24T16:45:18,985 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-24T16:45:18,985 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-24T16:45:18,985 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-24T16:45:18,985 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-24T16:45:18,985 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-24T16:45:18,986 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:33045 2024-11-24T16:45:18,987 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:33045 connecting to ZooKeeper ensemble=127.0.0.1:63542 2024-11-24T16:45:18,987 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-24T16:45:18,987 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-24T16:45:18,992 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:330450x0, quorum=127.0.0.1:63542, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-24T16:45:18,993 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:33045-0x1014ade23570001 connected 2024-11-24T16:45:18,993 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:33045-0x1014ade23570001, quorum=127.0.0.1:63542, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-24T16:45:18,993 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-24T16:45:18,994 INFO [Time-limited test {}] bucket.FileIOEngine(99): Allocating cache 400 MB, on the path:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/98608d84-f0c0-ab20-75c0-76a28c1904b5/bucket.cache 2024-11-24T16:45:18,994 INFO [Time-limited test {}] bucket.BucketCache(364): Instantiating BucketCache with acceptableFactor: 0.95, minFactor: 0.85, extraFreeFactor: 0.1, singleFactor: 0.25, multiFactor: 0.5, memoryFactor: 0.25, useStrongRef: false 2024-11-24T16:45:18,994 INFO [Time-limited test {}] bucket.BucketCache(478): Starting BucketCachePersisterThread 2024-11-24T16:45:18,994 INFO [Time-limited test {}] bucket.BucketCachePersister(38): BucketCachePersister started with interval: 100 2024-11-24T16:45:18,995 INFO [Thread-118 {}] bucket.BucketCache(1530): Started retrieving bucket cache from file 2024-11-24T16:45:18,995 WARN [Thread-118 {}] bucket.BucketCache(1533): Persistence file missing! It's ok if it's first run after enabling persistent cache. 2024-11-24T16:45:18,995 INFO [Thread-118 {}] bucket.BucketAllocator(343): Cache totalSize=418148352, buckets=199, bucket capacity=2101248=(4*525312)=(FEWEST_ITEMS_IN_BUCKET*(largest configured bucketcache size)) 2024-11-24T16:45:18,995 INFO [Thread-118 {}] bucket.BucketCache(413): Persistent bucket cache recovery from /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/98608d84-f0c0-ab20-75c0-76a28c1904b5/bucket.persistence is complete. 2024-11-24T16:45:18,996 INFO [Time-limited test {}] bucket.BucketCache(403): Started bucket cache; ioengine=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/98608d84-f0c0-ab20-75c0-76a28c1904b5/bucket.cache, capacity=400 MB, blockSize=64 KB, writerThreadNum=3, writerQLen=64, persistencePath=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/98608d84-f0c0-ab20-75c0-76a28c1904b5/bucket.persistence, bucketAllocator=org.apache.hadoop.hbase.io.hfile.bucket.BucketAllocator 2024-11-24T16:45:18,997 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-24T16:45:18,998 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:33045-0x1014ade23570001, quorum=127.0.0.1:63542, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-24T16:45:19,000 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:33045-0x1014ade23570001, quorum=127.0.0.1:63542, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-24T16:45:19,004 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=33045 2024-11-24T16:45:19,005 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=33045 2024-11-24T16:45:19,005 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=33045 2024-11-24T16:45:19,007 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=33045 2024-11-24T16:45:19,009 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=33045 2024-11-24T16:45:19,025 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/e335232f7491:0 server-side Connection retries=45 2024-11-24T16:45:19,026 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-24T16:45:19,026 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-24T16:45:19,026 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-24T16:45:19,026 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-24T16:45:19,026 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-24T16:45:19,026 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-24T16:45:19,026 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-24T16:45:19,027 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:42963 2024-11-24T16:45:19,029 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:42963 connecting to ZooKeeper ensemble=127.0.0.1:63542 2024-11-24T16:45:19,029 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-24T16:45:19,029 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-24T16:45:19,032 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:429630x0, quorum=127.0.0.1:63542, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-24T16:45:19,033 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:42963-0x1014ade23570002 connected 2024-11-24T16:45:19,033 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:42963-0x1014ade23570002, quorum=127.0.0.1:63542, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-24T16:45:19,033 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-24T16:45:19,034 INFO [Time-limited test {}] bucket.FileIOEngine(99): Allocating cache 400 MB, on the path:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/98608d84-f0c0-ab20-75c0-76a28c1904b5/bucket.cache 2024-11-24T16:45:19,034 INFO [Time-limited test {}] bucket.BucketCache(364): Instantiating BucketCache with acceptableFactor: 0.95, minFactor: 0.85, extraFreeFactor: 0.1, singleFactor: 0.25, multiFactor: 0.5, memoryFactor: 0.25, useStrongRef: false 2024-11-24T16:45:19,034 INFO [Time-limited test {}] bucket.BucketCache(478): Starting BucketCachePersisterThread 2024-11-24T16:45:19,034 INFO [Time-limited test {}] bucket.BucketCachePersister(38): BucketCachePersister started with interval: 100 2024-11-24T16:45:19,036 INFO [Thread-119 {}] bucket.BucketCache(1530): Started retrieving bucket cache from file 2024-11-24T16:45:19,036 WARN [Thread-119 {}] bucket.BucketCache(1533): Persistence file missing! It's ok if it's first run after enabling persistent cache. 2024-11-24T16:45:19,036 INFO [Thread-119 {}] bucket.BucketAllocator(343): Cache totalSize=418148352, buckets=199, bucket capacity=2101248=(4*525312)=(FEWEST_ITEMS_IN_BUCKET*(largest configured bucketcache size)) 2024-11-24T16:45:19,036 INFO [Time-limited test {}] bucket.BucketCache(403): Started bucket cache; ioengine=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/98608d84-f0c0-ab20-75c0-76a28c1904b5/bucket.cache, capacity=400 MB, blockSize=64 KB, writerThreadNum=3, writerQLen=64, persistencePath=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/98608d84-f0c0-ab20-75c0-76a28c1904b5/bucket.persistence, bucketAllocator=org.apache.hadoop.hbase.io.hfile.bucket.BucketAllocator 2024-11-24T16:45:19,036 INFO [Thread-119 {}] bucket.BucketCache(413): Persistent bucket cache recovery from /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/98608d84-f0c0-ab20-75c0-76a28c1904b5/bucket.persistence is complete. 2024-11-24T16:45:19,037 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-24T16:45:19,038 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:42963-0x1014ade23570002, quorum=127.0.0.1:63542, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-24T16:45:19,039 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:42963-0x1014ade23570002, quorum=127.0.0.1:63542, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-24T16:45:19,044 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=42963 2024-11-24T16:45:19,045 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=42963 2024-11-24T16:45:19,045 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=42963 2024-11-24T16:45:19,046 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=42963 2024-11-24T16:45:19,046 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=42963 2024-11-24T16:45:19,059 DEBUG [M:0;e335232f7491:36357 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;e335232f7491:36357 2024-11-24T16:45:19,059 INFO [master/e335232f7491:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/e335232f7491,36357,1732466718936 2024-11-24T16:45:19,061 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33045-0x1014ade23570001, quorum=127.0.0.1:63542, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-24T16:45:19,061 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36357-0x1014ade23570000, quorum=127.0.0.1:63542, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-24T16:45:19,061 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42963-0x1014ade23570002, quorum=127.0.0.1:63542, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-24T16:45:19,062 DEBUG [master/e335232f7491:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:36357-0x1014ade23570000, quorum=127.0.0.1:63542, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/e335232f7491,36357,1732466718936 2024-11-24T16:45:19,064 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33045-0x1014ade23570001, quorum=127.0.0.1:63542, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-24T16:45:19,065 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42963-0x1014ade23570002, quorum=127.0.0.1:63542, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-24T16:45:19,065 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36357-0x1014ade23570000, quorum=127.0.0.1:63542, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T16:45:19,065 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42963-0x1014ade23570002, quorum=127.0.0.1:63542, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T16:45:19,066 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33045-0x1014ade23570001, quorum=127.0.0.1:63542, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T16:45:19,066 DEBUG [master/e335232f7491:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:36357-0x1014ade23570000, quorum=127.0.0.1:63542, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-11-24T16:45:19,067 INFO [master/e335232f7491:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/e335232f7491,36357,1732466718936 from backup master directory 2024-11-24T16:45:19,069 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36357-0x1014ade23570000, quorum=127.0.0.1:63542, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/e335232f7491,36357,1732466718936 2024-11-24T16:45:19,069 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33045-0x1014ade23570001, quorum=127.0.0.1:63542, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-24T16:45:19,069 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42963-0x1014ade23570002, quorum=127.0.0.1:63542, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-24T16:45:19,069 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36357-0x1014ade23570000, quorum=127.0.0.1:63542, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-24T16:45:19,069 WARN [master/e335232f7491:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-24T16:45:19,069 INFO [master/e335232f7491:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=e335232f7491,36357,1732466718936 2024-11-24T16:45:19,071 DEBUG [master/e335232f7491:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9797cf4c-7cf0-a873-8ba7-f30267b4a2f7/39f4fdd4-f188-1d18-4286-5b0c4dc27cad/hbase.id] with ID: 8b6ab050-aca9-4038-b19c-0b3323afc835 2024-11-24T16:45:19,071 DEBUG [master/e335232f7491:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9797cf4c-7cf0-a873-8ba7-f30267b4a2f7/39f4fdd4-f188-1d18-4286-5b0c4dc27cad/.tmp/hbase.id 2024-11-24T16:45:19,077 DEBUG [master/e335232f7491:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9797cf4c-7cf0-a873-8ba7-f30267b4a2f7/39f4fdd4-f188-1d18-4286-5b0c4dc27cad/.tmp/hbase.id]:[file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9797cf4c-7cf0-a873-8ba7-f30267b4a2f7/39f4fdd4-f188-1d18-4286-5b0c4dc27cad/hbase.id] 2024-11-24T16:45:19,114 DEBUG [master/e335232f7491:0:becomeActiveMaster {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-24T16:45:19,114 INFO [master/e335232f7491:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem. 2024-11-24T16:45:19,116 INFO [master/e335232f7491:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 2ms. 2024-11-24T16:45:19,118 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36357-0x1014ade23570000, quorum=127.0.0.1:63542, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T16:45:19,118 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42963-0x1014ade23570002, quorum=127.0.0.1:63542, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T16:45:19,118 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33045-0x1014ade23570001, quorum=127.0.0.1:63542, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T16:45:19,123 INFO [master/e335232f7491:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-24T16:45:19,124 INFO [master/e335232f7491:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-11-24T16:45:19,124 INFO [master/e335232f7491:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-24T16:45:19,145 INFO [master/e335232f7491:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9797cf4c-7cf0-a873-8ba7-f30267b4a2f7/39f4fdd4-f188-1d18-4286-5b0c4dc27cad/MasterData/data/master/store 2024-11-24T16:45:19,152 DEBUG [master/e335232f7491:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T16:45:19,152 DEBUG [master/e335232f7491:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-24T16:45:19,152 INFO [master/e335232f7491:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-24T16:45:19,152 DEBUG [master/e335232f7491:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-24T16:45:19,152 DEBUG [master/e335232f7491:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-24T16:45:19,152 DEBUG [master/e335232f7491:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-24T16:45:19,152 INFO [master/e335232f7491:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-24T16:45:19,152 DEBUG [master/e335232f7491:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1732466719152Disabling compacts and flushes for region at 1732466719152Disabling writes for close at 1732466719152Writing region close event to WAL at 1732466719152Closed at 1732466719152 2024-11-24T16:45:19,152 WARN [master/e335232f7491:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9797cf4c-7cf0-a873-8ba7-f30267b4a2f7/39f4fdd4-f188-1d18-4286-5b0c4dc27cad/MasterData/data/master/store/.initializing 2024-11-24T16:45:19,152 DEBUG [master/e335232f7491:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9797cf4c-7cf0-a873-8ba7-f30267b4a2f7/39f4fdd4-f188-1d18-4286-5b0c4dc27cad/MasterData/WALs/e335232f7491,36357,1732466718936 2024-11-24T16:45:19,158 INFO [master/e335232f7491:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-24T16:45:19,161 INFO [master/e335232f7491:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=e335232f7491%2C36357%2C1732466718936, suffix=, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9797cf4c-7cf0-a873-8ba7-f30267b4a2f7/39f4fdd4-f188-1d18-4286-5b0c4dc27cad/MasterData/WALs/e335232f7491,36357,1732466718936, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9797cf4c-7cf0-a873-8ba7-f30267b4a2f7/39f4fdd4-f188-1d18-4286-5b0c4dc27cad/MasterData/oldWALs, maxLogs=10 2024-11-24T16:45:19,179 INFO [master/e335232f7491:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9797cf4c-7cf0-a873-8ba7-f30267b4a2f7/39f4fdd4-f188-1d18-4286-5b0c4dc27cad/MasterData/WALs/e335232f7491,36357,1732466718936/e335232f7491%2C36357%2C1732466718936.1732466719161 2024-11-24T16:45:19,179 DEBUG [master/e335232f7491:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-11-24T16:45:19,179 DEBUG [master/e335232f7491:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-11-24T16:45:19,180 DEBUG [master/e335232f7491:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T16:45:19,180 DEBUG [master/e335232f7491:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-11-24T16:45:19,180 DEBUG [master/e335232f7491:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-11-24T16:45:19,181 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-11-24T16:45:19,185 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-11-24T16:45:19,185 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:45:19,185 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-24T16:45:19,186 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-11-24T16:45:19,189 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-11-24T16:45:19,189 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:45:19,189 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T16:45:19,189 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-11-24T16:45:19,193 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-11-24T16:45:19,193 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:45:19,193 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T16:45:19,193 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-11-24T16:45:19,196 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-11-24T16:45:19,196 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:45:19,196 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T16:45:19,197 DEBUG [master/e335232f7491:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682 2024-11-24T16:45:19,197 DEBUG [master/e335232f7491:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9797cf4c-7cf0-a873-8ba7-f30267b4a2f7/39f4fdd4-f188-1d18-4286-5b0c4dc27cad/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-24T16:45:19,197 DEBUG [master/e335232f7491:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9797cf4c-7cf0-a873-8ba7-f30267b4a2f7/39f4fdd4-f188-1d18-4286-5b0c4dc27cad/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-24T16:45:19,197 DEBUG [master/e335232f7491:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682 2024-11-24T16:45:19,197 DEBUG [master/e335232f7491:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682 2024-11-24T16:45:19,198 DEBUG [master/e335232f7491:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-24T16:45:19,198 DEBUG [master/e335232f7491:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-11-24T16:45:19,207 DEBUG [master/e335232f7491:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9797cf4c-7cf0-a873-8ba7-f30267b4a2f7/39f4fdd4-f188-1d18-4286-5b0c4dc27cad/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-24T16:45:19,207 INFO [master/e335232f7491:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=65782286, jitterRate=-0.019767552614212036}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-24T16:45:19,208 DEBUG [master/e335232f7491:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682: Writing region info on filesystem at 1732466719180Initializing all the Stores at 1732466719180Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732466719180Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732466719180Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732466719180Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732466719180Cleaning up temporary data from old regions at 1732466719198 (+18 ms)Region opened successfully at 1732466719208 (+10 ms) 2024-11-24T16:45:19,209 INFO [master/e335232f7491:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-11-24T16:45:19,212 DEBUG [master/e335232f7491:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6cb6921, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=e335232f7491/172.17.0.2:0 2024-11-24T16:45:19,213 INFO [master/e335232f7491:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating... 2024-11-24T16:45:19,214 INFO [master/e335232f7491:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5 2024-11-24T16:45:19,214 INFO [master/e335232f7491:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50 2024-11-24T16:45:19,214 INFO [master/e335232f7491:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-24T16:45:19,214 INFO [master/e335232f7491:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-24T16:45:19,215 INFO [master/e335232f7491:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 0 msec 2024-11-24T16:45:19,215 INFO [master/e335232f7491:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-11-24T16:45:19,216 INFO [master/e335232f7491:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2024-11-24T16:45:19,217 DEBUG [master/e335232f7491:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:36357-0x1014ade23570000, quorum=127.0.0.1:63542, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error) 2024-11-24T16:45:19,219 DEBUG [master/e335232f7491:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false 2024-11-24T16:45:19,219 INFO [master/e335232f7491:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2024-11-24T16:45:19,220 DEBUG [master/e335232f7491:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:36357-0x1014ade23570000, quorum=127.0.0.1:63542, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error) 2024-11-24T16:45:19,221 DEBUG [master/e335232f7491:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false 2024-11-24T16:45:19,222 INFO [master/e335232f7491:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2024-11-24T16:45:19,222 DEBUG [master/e335232f7491:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:36357-0x1014ade23570000, quorum=127.0.0.1:63542, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error) 2024-11-24T16:45:19,225 DEBUG [master/e335232f7491:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false 2024-11-24T16:45:19,226 DEBUG [master/e335232f7491:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:36357-0x1014ade23570000, quorum=127.0.0.1:63542, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error) 2024-11-24T16:45:19,227 DEBUG [master/e335232f7491:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false 2024-11-24T16:45:19,229 DEBUG [master/e335232f7491:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:36357-0x1014ade23570000, quorum=127.0.0.1:63542, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error) 2024-11-24T16:45:19,231 DEBUG [master/e335232f7491:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false 2024-11-24T16:45:19,232 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36357-0x1014ade23570000, quorum=127.0.0.1:63542, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-24T16:45:19,233 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33045-0x1014ade23570001, quorum=127.0.0.1:63542, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-24T16:45:19,233 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36357-0x1014ade23570000, quorum=127.0.0.1:63542, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T16:45:19,233 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33045-0x1014ade23570001, quorum=127.0.0.1:63542, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T16:45:19,233 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42963-0x1014ade23570002, quorum=127.0.0.1:63542, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-24T16:45:19,233 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42963-0x1014ade23570002, quorum=127.0.0.1:63542, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T16:45:19,233 INFO [master/e335232f7491:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=e335232f7491,36357,1732466718936, sessionid=0x1014ade23570000, setting cluster-up flag (Was=false) 2024-11-24T16:45:19,237 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36357-0x1014ade23570000, quorum=127.0.0.1:63542, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T16:45:19,237 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33045-0x1014ade23570001, quorum=127.0.0.1:63542, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T16:45:19,237 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42963-0x1014ade23570002, quorum=127.0.0.1:63542, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T16:45:19,243 DEBUG [master/e335232f7491:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort 2024-11-24T16:45:19,244 DEBUG [master/e335232f7491:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=e335232f7491,36357,1732466718936 2024-11-24T16:45:19,246 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36357-0x1014ade23570000, quorum=127.0.0.1:63542, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T16:45:19,246 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42963-0x1014ade23570002, quorum=127.0.0.1:63542, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T16:45:19,246 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33045-0x1014ade23570001, quorum=127.0.0.1:63542, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T16:45:19,253 DEBUG [master/e335232f7491:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort 2024-11-24T16:45:19,254 DEBUG [master/e335232f7491:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=e335232f7491,36357,1732466718936 2024-11-24T16:45:19,255 INFO [master/e335232f7491:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9797cf4c-7cf0-a873-8ba7-f30267b4a2f7/39f4fdd4-f188-1d18-4286-5b0c4dc27cad/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again 2024-11-24T16:45:19,256 DEBUG [master/e335232f7491:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-24T16:45:19,256 INFO [master/e335232f7491:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2 2024-11-24T16:45:19,256 INFO [master/e335232f7491:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2024-11-24T16:45:19,257 DEBUG [master/e335232f7491:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: e335232f7491,36357,1732466718936 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2024-11-24T16:45:19,258 DEBUG [master/e335232f7491:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/e335232f7491:0, corePoolSize=5, maxPoolSize=5 2024-11-24T16:45:19,258 DEBUG [master/e335232f7491:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/e335232f7491:0, corePoolSize=5, maxPoolSize=5 2024-11-24T16:45:19,259 DEBUG [master/e335232f7491:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/e335232f7491:0, corePoolSize=5, maxPoolSize=5 2024-11-24T16:45:19,259 DEBUG [master/e335232f7491:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/e335232f7491:0, corePoolSize=5, maxPoolSize=5 2024-11-24T16:45:19,259 DEBUG [master/e335232f7491:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/e335232f7491:0, corePoolSize=10, maxPoolSize=10 2024-11-24T16:45:19,259 DEBUG [master/e335232f7491:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/e335232f7491:0, corePoolSize=1, maxPoolSize=1 2024-11-24T16:45:19,259 DEBUG [master/e335232f7491:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/e335232f7491:0, corePoolSize=2, maxPoolSize=2 2024-11-24T16:45:19,259 DEBUG [master/e335232f7491:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/e335232f7491:0, corePoolSize=1, maxPoolSize=1 2024-11-24T16:45:19,262 INFO [master/e335232f7491:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732466749262 2024-11-24T16:45:19,262 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-24T16:45:19,262 INFO [master/e335232f7491:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2024-11-24T16:45:19,262 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region 2024-11-24T16:45:19,262 INFO [master/e335232f7491:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2024-11-24T16:45:19,262 INFO [master/e335232f7491:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2024-11-24T16:45:19,262 INFO [master/e335232f7491:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2024-11-24T16:45:19,262 INFO [master/e335232f7491:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2024-11-24T16:45:19,262 INFO [master/e335232f7491:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2024-11-24T16:45:19,263 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:45:19,263 INFO [master/e335232f7491:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-24T16:45:19,263 INFO [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-11-24T16:45:19,264 INFO [master/e335232f7491:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2024-11-24T16:45:19,264 INFO [master/e335232f7491:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2024-11-24T16:45:19,264 INFO [master/e335232f7491:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2024-11-24T16:45:19,264 INFO [master/e335232f7491:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2024-11-24T16:45:19,264 INFO [master/e335232f7491:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2024-11-24T16:45:19,264 DEBUG [master/e335232f7491:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/e335232f7491:0:becomeActiveMaster-HFileCleaner.large.0-1732466719264,5,FailOnTimeoutGroup] 2024-11-24T16:45:19,265 DEBUG [master/e335232f7491:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/e335232f7491:0:becomeActiveMaster-HFileCleaner.small.0-1732466719265,5,FailOnTimeoutGroup] 2024-11-24T16:45:19,265 INFO [master/e335232f7491:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-24T16:45:19,265 INFO [master/e335232f7491:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2024-11-24T16:45:19,265 INFO [master/e335232f7491:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2024-11-24T16:45:19,265 INFO [master/e335232f7491:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2024-11-24T16:45:19,277 INFO [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9797cf4c-7cf0-a873-8ba7-f30267b4a2f7/39f4fdd4-f188-1d18-4286-5b0c4dc27cad/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321 2024-11-24T16:45:19,277 INFO [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9797cf4c-7cf0-a873-8ba7-f30267b4a2f7/39f4fdd4-f188-1d18-4286-5b0c4dc27cad 2024-11-24T16:45:19,283 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T16:45:19,284 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family info of region 1588230740 2024-11-24T16:45:19,288 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-24T16:45:19,288 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:45:19,288 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-24T16:45:19,288 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family ns of region 1588230740 2024-11-24T16:45:19,291 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-24T16:45:19,291 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:45:19,292 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-24T16:45:19,292 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family rep_barrier of region 1588230740 2024-11-24T16:45:19,295 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-24T16:45:19,295 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:45:19,295 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-24T16:45:19,295 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family table of region 1588230740 2024-11-24T16:45:19,299 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-24T16:45:19,299 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:45:19,299 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-24T16:45:19,299 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-24T16:45:19,300 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9797cf4c-7cf0-a873-8ba7-f30267b4a2f7/39f4fdd4-f188-1d18-4286-5b0c4dc27cad/data/hbase/meta/1588230740 2024-11-24T16:45:19,300 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9797cf4c-7cf0-a873-8ba7-f30267b4a2f7/39f4fdd4-f188-1d18-4286-5b0c4dc27cad/data/hbase/meta/1588230740 2024-11-24T16:45:19,300 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-24T16:45:19,300 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-24T16:45:19,301 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-24T16:45:19,301 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-24T16:45:19,307 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9797cf4c-7cf0-a873-8ba7-f30267b4a2f7/39f4fdd4-f188-1d18-4286-5b0c4dc27cad/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-24T16:45:19,307 INFO [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=64061426, jitterRate=-0.045410364866256714}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-24T16:45:19,308 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740: Writing region info on filesystem at 1732466719284Initializing all the Stores at 1732466719284Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732466719284Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732466719284Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732466719284Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732466719284Cleaning up temporary data from old regions at 1732466719300 (+16 ms)Region opened successfully at 1732466719308 (+8 ms) 2024-11-24T16:45:19,308 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-24T16:45:19,308 INFO [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-24T16:45:19,308 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-24T16:45:19,308 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-24T16:45:19,308 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-24T16:45:19,309 INFO [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-24T16:45:19,309 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1732466719308Disabling compacts and flushes for region at 1732466719308Disabling writes for close at 1732466719308Writing region close event to WAL at 1732466719309 (+1 ms)Closed at 1732466719309 2024-11-24T16:45:19,310 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-24T16:45:19,310 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta 2024-11-24T16:45:19,310 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-11-24T16:45:19,311 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-24T16:45:19,311 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2024-11-24T16:45:19,349 INFO [RS:1;e335232f7491:42963 {}] regionserver.HRegionServer(746): ClusterId : 8b6ab050-aca9-4038-b19c-0b3323afc835 2024-11-24T16:45:19,349 INFO [RS:0;e335232f7491:33045 {}] regionserver.HRegionServer(746): ClusterId : 8b6ab050-aca9-4038-b19c-0b3323afc835 2024-11-24T16:45:19,350 DEBUG [RS:0;e335232f7491:33045 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-24T16:45:19,350 DEBUG [RS:1;e335232f7491:42963 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-24T16:45:19,352 DEBUG [RS:0;e335232f7491:33045 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-24T16:45:19,352 DEBUG [RS:0;e335232f7491:33045 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-24T16:45:19,352 DEBUG [RS:1;e335232f7491:42963 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-24T16:45:19,352 DEBUG [RS:1;e335232f7491:42963 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-24T16:45:19,355 DEBUG [RS:0;e335232f7491:33045 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-24T16:45:19,356 DEBUG [RS:1;e335232f7491:42963 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-24T16:45:19,356 DEBUG [RS:1;e335232f7491:42963 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3c3d5747, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=e335232f7491/172.17.0.2:0 2024-11-24T16:45:19,356 DEBUG [RS:0;e335232f7491:33045 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@79bcd3db, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=e335232f7491/172.17.0.2:0 2024-11-24T16:45:19,368 DEBUG [RS:1;e335232f7491:42963 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:1;e335232f7491:42963 2024-11-24T16:45:19,368 INFO [RS:1;e335232f7491:42963 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-24T16:45:19,369 INFO [RS:1;e335232f7491:42963 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-24T16:45:19,369 DEBUG [RS:1;e335232f7491:42963 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-24T16:45:19,370 INFO [RS:1;e335232f7491:42963 {}] regionserver.HRegionServer(2659): reportForDuty to master=e335232f7491,36357,1732466718936 with port=42963, startcode=1732466719025 2024-11-24T16:45:19,370 DEBUG [RS:1;e335232f7491:42963 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-24T16:45:19,372 DEBUG [RS:0;e335232f7491:33045 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;e335232f7491:33045 2024-11-24T16:45:19,372 INFO [RS:0;e335232f7491:33045 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-24T16:45:19,372 INFO [RS:0;e335232f7491:33045 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-24T16:45:19,372 DEBUG [RS:0;e335232f7491:33045 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-24T16:45:19,373 INFO [RS:0;e335232f7491:33045 {}] regionserver.HRegionServer(2659): reportForDuty to master=e335232f7491,36357,1732466718936 with port=33045, startcode=1732466718984 2024-11-24T16:45:19,373 DEBUG [RS:0;e335232f7491:33045 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-24T16:45:19,373 INFO [HMaster-EventLoopGroup-6-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40185, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=RegionServerStatusService 2024-11-24T16:45:19,374 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=36357 {}] master.ServerManager(363): Checking decommissioned status of RegionServer e335232f7491,42963,1732466719025 2024-11-24T16:45:19,374 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=36357 {}] master.ServerManager(517): Registering regionserver=e335232f7491,42963,1732466719025 2024-11-24T16:45:19,375 INFO [HMaster-EventLoopGroup-6-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57211, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=RegionServerStatusService 2024-11-24T16:45:19,376 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=36357 {}] master.ServerManager(363): Checking decommissioned status of RegionServer e335232f7491,33045,1732466718984 2024-11-24T16:45:19,376 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=36357 {}] master.ServerManager(517): Registering regionserver=e335232f7491,33045,1732466718984 2024-11-24T16:45:19,376 DEBUG [RS:1;e335232f7491:42963 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9797cf4c-7cf0-a873-8ba7-f30267b4a2f7/39f4fdd4-f188-1d18-4286-5b0c4dc27cad 2024-11-24T16:45:19,376 DEBUG [RS:1;e335232f7491:42963 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=file:/ 2024-11-24T16:45:19,376 DEBUG [RS:1;e335232f7491:42963 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-24T16:45:19,377 DEBUG [RS:0;e335232f7491:33045 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9797cf4c-7cf0-a873-8ba7-f30267b4a2f7/39f4fdd4-f188-1d18-4286-5b0c4dc27cad 2024-11-24T16:45:19,377 DEBUG [RS:0;e335232f7491:33045 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=file:/ 2024-11-24T16:45:19,377 DEBUG [RS:0;e335232f7491:33045 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-24T16:45:19,378 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36357-0x1014ade23570000, quorum=127.0.0.1:63542, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-24T16:45:19,382 DEBUG [RS:1;e335232f7491:42963 {}] zookeeper.ZKUtil(111): regionserver:42963-0x1014ade23570002, quorum=127.0.0.1:63542, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/e335232f7491,42963,1732466719025 2024-11-24T16:45:19,382 WARN [RS:1;e335232f7491:42963 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-24T16:45:19,382 INFO [RS:1;e335232f7491:42963 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-24T16:45:19,382 DEBUG [RS:1;e335232f7491:42963 {}] regionserver.HRegionServer(1793): logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9797cf4c-7cf0-a873-8ba7-f30267b4a2f7/39f4fdd4-f188-1d18-4286-5b0c4dc27cad/WALs/e335232f7491,42963,1732466719025 2024-11-24T16:45:19,382 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [e335232f7491,42963,1732466719025] 2024-11-24T16:45:19,382 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [e335232f7491,33045,1732466718984] 2024-11-24T16:45:19,382 DEBUG [RS:0;e335232f7491:33045 {}] zookeeper.ZKUtil(111): regionserver:33045-0x1014ade23570001, quorum=127.0.0.1:63542, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/e335232f7491,33045,1732466718984 2024-11-24T16:45:19,382 WARN [RS:0;e335232f7491:33045 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-24T16:45:19,382 INFO [RS:0;e335232f7491:33045 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-24T16:45:19,383 DEBUG [RS:0;e335232f7491:33045 {}] regionserver.HRegionServer(1793): logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9797cf4c-7cf0-a873-8ba7-f30267b4a2f7/39f4fdd4-f188-1d18-4286-5b0c4dc27cad/WALs/e335232f7491,33045,1732466718984 2024-11-24T16:45:19,387 INFO [RS:1;e335232f7491:42963 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-24T16:45:19,390 INFO [RS:1;e335232f7491:42963 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-24T16:45:19,390 INFO [RS:0;e335232f7491:33045 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-24T16:45:19,391 INFO [RS:1;e335232f7491:42963 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-24T16:45:19,391 INFO [RS:1;e335232f7491:42963 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-24T16:45:19,392 INFO [RS:1;e335232f7491:42963 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-24T16:45:19,392 INFO [RS:1;e335232f7491:42963 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-24T16:45:19,392 INFO [RS:1;e335232f7491:42963 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-24T16:45:19,392 DEBUG [RS:1;e335232f7491:42963 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/e335232f7491:0, corePoolSize=1, maxPoolSize=1 2024-11-24T16:45:19,393 DEBUG [RS:1;e335232f7491:42963 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/e335232f7491:0, corePoolSize=1, maxPoolSize=1 2024-11-24T16:45:19,393 DEBUG [RS:1;e335232f7491:42963 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/e335232f7491:0, corePoolSize=1, maxPoolSize=1 2024-11-24T16:45:19,393 DEBUG [RS:1;e335232f7491:42963 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/e335232f7491:0, corePoolSize=1, maxPoolSize=1 2024-11-24T16:45:19,393 DEBUG [RS:1;e335232f7491:42963 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/e335232f7491:0, corePoolSize=1, maxPoolSize=1 2024-11-24T16:45:19,393 DEBUG [RS:1;e335232f7491:42963 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/e335232f7491:0, corePoolSize=2, maxPoolSize=2 2024-11-24T16:45:19,393 DEBUG [RS:1;e335232f7491:42963 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/e335232f7491:0, corePoolSize=1, maxPoolSize=1 2024-11-24T16:45:19,393 INFO [RS:0;e335232f7491:33045 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-24T16:45:19,393 DEBUG [RS:1;e335232f7491:42963 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/e335232f7491:0, corePoolSize=1, maxPoolSize=1 2024-11-24T16:45:19,393 DEBUG [RS:1;e335232f7491:42963 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/e335232f7491:0, corePoolSize=1, maxPoolSize=1 2024-11-24T16:45:19,393 DEBUG [RS:1;e335232f7491:42963 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/e335232f7491:0, corePoolSize=1, maxPoolSize=1 2024-11-24T16:45:19,393 DEBUG [RS:1;e335232f7491:42963 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/e335232f7491:0, corePoolSize=1, maxPoolSize=1 2024-11-24T16:45:19,394 DEBUG [RS:1;e335232f7491:42963 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/e335232f7491:0, corePoolSize=1, maxPoolSize=1 2024-11-24T16:45:19,394 DEBUG [RS:1;e335232f7491:42963 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/e335232f7491:0, corePoolSize=3, maxPoolSize=3 2024-11-24T16:45:19,394 DEBUG [RS:1;e335232f7491:42963 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0, corePoolSize=3, maxPoolSize=3 2024-11-24T16:45:19,394 INFO [RS:0;e335232f7491:33045 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-24T16:45:19,394 INFO [RS:0;e335232f7491:33045 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-24T16:45:19,394 INFO [RS:1;e335232f7491:42963 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-24T16:45:19,394 INFO [RS:1;e335232f7491:42963 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-24T16:45:19,394 INFO [RS:1;e335232f7491:42963 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-24T16:45:19,394 INFO [RS:0;e335232f7491:33045 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-24T16:45:19,394 INFO [RS:1;e335232f7491:42963 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-24T16:45:19,395 INFO [RS:1;e335232f7491:42963 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-24T16:45:19,395 INFO [RS:1;e335232f7491:42963 {}] hbase.ChoreService(168): Chore ScheduledChore name=e335232f7491,42963,1732466719025-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-24T16:45:19,395 INFO [RS:0;e335232f7491:33045 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-24T16:45:19,395 INFO [RS:0;e335232f7491:33045 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-24T16:45:19,395 DEBUG [RS:0;e335232f7491:33045 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/e335232f7491:0, corePoolSize=1, maxPoolSize=1 2024-11-24T16:45:19,395 DEBUG [RS:0;e335232f7491:33045 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/e335232f7491:0, corePoolSize=1, maxPoolSize=1 2024-11-24T16:45:19,395 DEBUG [RS:0;e335232f7491:33045 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/e335232f7491:0, corePoolSize=1, maxPoolSize=1 2024-11-24T16:45:19,395 DEBUG [RS:0;e335232f7491:33045 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/e335232f7491:0, corePoolSize=1, maxPoolSize=1 2024-11-24T16:45:19,396 DEBUG [RS:0;e335232f7491:33045 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/e335232f7491:0, corePoolSize=1, maxPoolSize=1 2024-11-24T16:45:19,396 DEBUG [RS:0;e335232f7491:33045 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/e335232f7491:0, corePoolSize=2, maxPoolSize=2 2024-11-24T16:45:19,396 DEBUG [RS:0;e335232f7491:33045 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/e335232f7491:0, corePoolSize=1, maxPoolSize=1 2024-11-24T16:45:19,396 DEBUG [RS:0;e335232f7491:33045 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/e335232f7491:0, corePoolSize=1, maxPoolSize=1 2024-11-24T16:45:19,396 DEBUG [RS:0;e335232f7491:33045 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/e335232f7491:0, corePoolSize=1, maxPoolSize=1 2024-11-24T16:45:19,396 DEBUG [RS:0;e335232f7491:33045 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/e335232f7491:0, corePoolSize=1, maxPoolSize=1 2024-11-24T16:45:19,396 DEBUG [RS:0;e335232f7491:33045 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/e335232f7491:0, corePoolSize=1, maxPoolSize=1 2024-11-24T16:45:19,396 DEBUG [RS:0;e335232f7491:33045 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/e335232f7491:0, corePoolSize=1, maxPoolSize=1 2024-11-24T16:45:19,396 DEBUG [RS:0;e335232f7491:33045 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/e335232f7491:0, corePoolSize=3, maxPoolSize=3 2024-11-24T16:45:19,396 DEBUG [RS:0;e335232f7491:33045 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0, corePoolSize=3, maxPoolSize=3 2024-11-24T16:45:19,398 INFO [RS:0;e335232f7491:33045 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-24T16:45:19,398 INFO [RS:0;e335232f7491:33045 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-24T16:45:19,398 INFO [RS:0;e335232f7491:33045 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-24T16:45:19,398 INFO [RS:0;e335232f7491:33045 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-24T16:45:19,398 INFO [RS:0;e335232f7491:33045 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-24T16:45:19,398 INFO [RS:0;e335232f7491:33045 {}] hbase.ChoreService(168): Chore ScheduledChore name=e335232f7491,33045,1732466718984-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-24T16:45:19,414 INFO [RS:1;e335232f7491:42963 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-24T16:45:19,414 INFO [RS:0;e335232f7491:33045 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-24T16:45:19,414 INFO [RS:1;e335232f7491:42963 {}] hbase.ChoreService(168): Chore ScheduledChore name=e335232f7491,42963,1732466719025-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-24T16:45:19,414 INFO [RS:0;e335232f7491:33045 {}] hbase.ChoreService(168): Chore ScheduledChore name=e335232f7491,33045,1732466718984-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-24T16:45:19,415 INFO [RS:1;e335232f7491:42963 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-24T16:45:19,415 INFO [RS:0;e335232f7491:33045 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-24T16:45:19,415 INFO [RS:1;e335232f7491:42963 {}] regionserver.Replication(171): e335232f7491,42963,1732466719025 started 2024-11-24T16:45:19,415 INFO [RS:0;e335232f7491:33045 {}] regionserver.Replication(171): e335232f7491,33045,1732466718984 started 2024-11-24T16:45:19,429 INFO [RS:0;e335232f7491:33045 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-24T16:45:19,429 INFO [RS:0;e335232f7491:33045 {}] regionserver.HRegionServer(1482): Serving as e335232f7491,33045,1732466718984, RpcServer on e335232f7491/172.17.0.2:33045, sessionid=0x1014ade23570001 2024-11-24T16:45:19,429 DEBUG [RS:0;e335232f7491:33045 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-24T16:45:19,429 DEBUG [RS:0;e335232f7491:33045 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager e335232f7491,33045,1732466718984 2024-11-24T16:45:19,429 DEBUG [RS:0;e335232f7491:33045 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'e335232f7491,33045,1732466718984' 2024-11-24T16:45:19,429 DEBUG [RS:0;e335232f7491:33045 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-24T16:45:19,429 INFO [RS:1;e335232f7491:42963 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-24T16:45:19,429 INFO [RS:1;e335232f7491:42963 {}] regionserver.HRegionServer(1482): Serving as e335232f7491,42963,1732466719025, RpcServer on e335232f7491/172.17.0.2:42963, sessionid=0x1014ade23570002 2024-11-24T16:45:19,430 DEBUG [RS:1;e335232f7491:42963 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-24T16:45:19,430 DEBUG [RS:1;e335232f7491:42963 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager e335232f7491,42963,1732466719025 2024-11-24T16:45:19,430 DEBUG [RS:1;e335232f7491:42963 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'e335232f7491,42963,1732466719025' 2024-11-24T16:45:19,430 DEBUG [RS:1;e335232f7491:42963 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-24T16:45:19,430 DEBUG [RS:0;e335232f7491:33045 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-24T16:45:19,430 DEBUG [RS:1;e335232f7491:42963 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-24T16:45:19,431 DEBUG [RS:0;e335232f7491:33045 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-24T16:45:19,431 DEBUG [RS:0;e335232f7491:33045 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-24T16:45:19,431 DEBUG [RS:0;e335232f7491:33045 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager e335232f7491,33045,1732466718984 2024-11-24T16:45:19,431 DEBUG [RS:0;e335232f7491:33045 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'e335232f7491,33045,1732466718984' 2024-11-24T16:45:19,431 DEBUG [RS:0;e335232f7491:33045 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-24T16:45:19,431 DEBUG [RS:1;e335232f7491:42963 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-24T16:45:19,431 DEBUG [RS:1;e335232f7491:42963 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-24T16:45:19,431 DEBUG [RS:1;e335232f7491:42963 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager e335232f7491,42963,1732466719025 2024-11-24T16:45:19,431 DEBUG [RS:1;e335232f7491:42963 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'e335232f7491,42963,1732466719025' 2024-11-24T16:45:19,431 DEBUG [RS:1;e335232f7491:42963 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-24T16:45:19,431 DEBUG [RS:0;e335232f7491:33045 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-24T16:45:19,432 DEBUG [RS:1;e335232f7491:42963 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-24T16:45:19,432 DEBUG [RS:0;e335232f7491:33045 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-24T16:45:19,432 INFO [RS:0;e335232f7491:33045 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-24T16:45:19,432 INFO [RS:0;e335232f7491:33045 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-24T16:45:19,432 DEBUG [RS:1;e335232f7491:42963 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-24T16:45:19,432 INFO [RS:1;e335232f7491:42963 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-24T16:45:19,432 INFO [RS:1;e335232f7491:42963 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-24T16:45:19,462 WARN [e335232f7491:36357 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-24T16:45:19,533 INFO [RS:0;e335232f7491:33045 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-24T16:45:19,533 INFO [RS:1;e335232f7491:42963 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-24T16:45:19,533 INFO [RS:0;e335232f7491:33045 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=e335232f7491%2C33045%2C1732466718984, suffix=, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9797cf4c-7cf0-a873-8ba7-f30267b4a2f7/39f4fdd4-f188-1d18-4286-5b0c4dc27cad/WALs/e335232f7491,33045,1732466718984, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9797cf4c-7cf0-a873-8ba7-f30267b4a2f7/39f4fdd4-f188-1d18-4286-5b0c4dc27cad/oldWALs, maxLogs=55 2024-11-24T16:45:19,534 INFO [RS:1;e335232f7491:42963 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=e335232f7491%2C42963%2C1732466719025, suffix=, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9797cf4c-7cf0-a873-8ba7-f30267b4a2f7/39f4fdd4-f188-1d18-4286-5b0c4dc27cad/WALs/e335232f7491,42963,1732466719025, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9797cf4c-7cf0-a873-8ba7-f30267b4a2f7/39f4fdd4-f188-1d18-4286-5b0c4dc27cad/oldWALs, maxLogs=55 2024-11-24T16:45:19,553 INFO [RS:0;e335232f7491:33045 {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9797cf4c-7cf0-a873-8ba7-f30267b4a2f7/39f4fdd4-f188-1d18-4286-5b0c4dc27cad/WALs/e335232f7491,33045,1732466718984/e335232f7491%2C33045%2C1732466718984.1732466719535 2024-11-24T16:45:19,553 DEBUG [RS:0;e335232f7491:33045 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-11-24T16:45:19,557 INFO [RS:1;e335232f7491:42963 {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9797cf4c-7cf0-a873-8ba7-f30267b4a2f7/39f4fdd4-f188-1d18-4286-5b0c4dc27cad/WALs/e335232f7491,42963,1732466719025/e335232f7491%2C42963%2C1732466719025.1732466719536 2024-11-24T16:45:19,557 DEBUG [RS:1;e335232f7491:42963 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-11-24T16:45:19,712 DEBUG [e335232f7491:36357 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=2, allServersCount=2 2024-11-24T16:45:19,712 DEBUG [e335232f7491:36357 {}] balancer.BalancerClusterState(204): Hosts are {e335232f7491=0} racks are {/default-rack=0} 2024-11-24T16:45:19,713 DEBUG [e335232f7491:36357 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-11-24T16:45:19,713 DEBUG [e335232f7491:36357 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-11-24T16:45:19,713 DEBUG [e335232f7491:36357 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-11-24T16:45:19,713 DEBUG [e335232f7491:36357 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-11-24T16:45:19,713 INFO [e335232f7491:36357 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-11-24T16:45:19,713 INFO [e335232f7491:36357 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-11-24T16:45:19,713 DEBUG [e335232f7491:36357 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-11-24T16:45:19,714 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=e335232f7491,42963,1732466719025 2024-11-24T16:45:19,715 INFO [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as e335232f7491,42963,1732466719025, state=OPENING 2024-11-24T16:45:19,717 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2024-11-24T16:45:19,719 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36357-0x1014ade23570000, quorum=127.0.0.1:63542, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T16:45:19,719 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33045-0x1014ade23570001, quorum=127.0.0.1:63542, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T16:45:19,719 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42963-0x1014ade23570002, quorum=127.0.0.1:63542, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T16:45:19,720 DEBUG [PEWorker-3 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-24T16:45:19,720 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=e335232f7491,42963,1732466719025}] 2024-11-24T16:45:19,720 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-24T16:45:19,720 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-24T16:45:19,720 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-24T16:45:19,873 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-24T16:45:19,875 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-8-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43085, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-24T16:45:19,877 INFO [RS_OPEN_META-regionserver/e335232f7491:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-11-24T16:45:19,878 INFO [RS_OPEN_META-regionserver/e335232f7491:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-24T16:45:19,878 INFO [RS_OPEN_META-regionserver/e335232f7491:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-11-24T16:45:19,879 INFO [RS_OPEN_META-regionserver/e335232f7491:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=e335232f7491%2C42963%2C1732466719025.meta, suffix=.meta, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9797cf4c-7cf0-a873-8ba7-f30267b4a2f7/39f4fdd4-f188-1d18-4286-5b0c4dc27cad/WALs/e335232f7491,42963,1732466719025, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9797cf4c-7cf0-a873-8ba7-f30267b4a2f7/39f4fdd4-f188-1d18-4286-5b0c4dc27cad/oldWALs, maxLogs=55 2024-11-24T16:45:19,898 INFO [RS_OPEN_META-regionserver/e335232f7491:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9797cf4c-7cf0-a873-8ba7-f30267b4a2f7/39f4fdd4-f188-1d18-4286-5b0c4dc27cad/WALs/e335232f7491,42963,1732466719025/e335232f7491%2C42963%2C1732466719025.meta.1732466719879.meta 2024-11-24T16:45:19,898 DEBUG [RS_OPEN_META-regionserver/e335232f7491:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-11-24T16:45:19,898 DEBUG [RS_OPEN_META-regionserver/e335232f7491:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-11-24T16:45:19,899 DEBUG [RS_OPEN_META-regionserver/e335232f7491:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-11-24T16:45:19,899 DEBUG [RS_OPEN_META-regionserver/e335232f7491:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-11-24T16:45:19,899 INFO [RS_OPEN_META-regionserver/e335232f7491:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-11-24T16:45:19,899 DEBUG [RS_OPEN_META-regionserver/e335232f7491:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-11-24T16:45:19,899 DEBUG [RS_OPEN_META-regionserver/e335232f7491:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T16:45:19,899 DEBUG [RS_OPEN_META-regionserver/e335232f7491:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-11-24T16:45:19,900 DEBUG [RS_OPEN_META-regionserver/e335232f7491:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-11-24T16:45:19,900 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family info of region 1588230740 2024-11-24T16:45:19,901 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-24T16:45:19,901 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:45:19,901 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-24T16:45:19,901 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family ns of region 1588230740 2024-11-24T16:45:19,902 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-24T16:45:19,902 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:45:19,902 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-24T16:45:19,902 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family rep_barrier of region 1588230740 2024-11-24T16:45:19,903 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-24T16:45:19,903 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:45:19,903 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-24T16:45:19,903 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family table of region 1588230740 2024-11-24T16:45:19,904 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-24T16:45:19,904 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:45:19,904 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-24T16:45:19,905 DEBUG [RS_OPEN_META-regionserver/e335232f7491:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-24T16:45:19,905 DEBUG [RS_OPEN_META-regionserver/e335232f7491:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9797cf4c-7cf0-a873-8ba7-f30267b4a2f7/39f4fdd4-f188-1d18-4286-5b0c4dc27cad/data/hbase/meta/1588230740 2024-11-24T16:45:19,906 DEBUG [RS_OPEN_META-regionserver/e335232f7491:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9797cf4c-7cf0-a873-8ba7-f30267b4a2f7/39f4fdd4-f188-1d18-4286-5b0c4dc27cad/data/hbase/meta/1588230740 2024-11-24T16:45:19,906 DEBUG [RS_OPEN_META-regionserver/e335232f7491:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-24T16:45:19,906 DEBUG [RS_OPEN_META-regionserver/e335232f7491:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-24T16:45:19,906 DEBUG [RS_OPEN_META-regionserver/e335232f7491:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-24T16:45:19,907 DEBUG [RS_OPEN_META-regionserver/e335232f7491:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-24T16:45:19,907 INFO [RS_OPEN_META-regionserver/e335232f7491:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=64792570, jitterRate=-0.03451547026634216}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-24T16:45:19,907 DEBUG [RS_OPEN_META-regionserver/e335232f7491:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-11-24T16:45:19,908 DEBUG [RS_OPEN_META-regionserver/e335232f7491:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1732466719900Writing region info on filesystem at 1732466719900Initializing all the Stores at 1732466719900Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732466719900Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732466719900Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732466719900Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732466719900Cleaning up temporary data from old regions at 1732466719906 (+6 ms)Running coprocessor post-open hooks at 1732466719907 (+1 ms)Region opened successfully at 1732466719908 (+1 ms) 2024-11-24T16:45:19,909 INFO [RS_OPEN_META-regionserver/e335232f7491:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1732466719872 2024-11-24T16:45:19,912 DEBUG [RS_OPEN_META-regionserver/e335232f7491:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-11-24T16:45:19,912 INFO [RS_OPEN_META-regionserver/e335232f7491:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-11-24T16:45:19,912 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=e335232f7491,42963,1732466719025 2024-11-24T16:45:19,913 INFO [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as e335232f7491,42963,1732466719025, state=OPEN 2024-11-24T16:45:19,914 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36357-0x1014ade23570000, quorum=127.0.0.1:63542, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-24T16:45:19,914 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33045-0x1014ade23570001, quorum=127.0.0.1:63542, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-24T16:45:19,914 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42963-0x1014ade23570002, quorum=127.0.0.1:63542, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-24T16:45:19,914 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=e335232f7491,42963,1732466719025 2024-11-24T16:45:19,914 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-24T16:45:19,915 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-24T16:45:19,915 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-24T16:45:19,916 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2 2024-11-24T16:45:19,916 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=e335232f7491,42963,1732466719025 in 194 msec 2024-11-24T16:45:19,918 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1 2024-11-24T16:45:19,918 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 606 msec 2024-11-24T16:45:19,919 DEBUG [PEWorker-2 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-24T16:45:19,919 INFO [PEWorker-2 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces 2024-11-24T16:45:19,922 DEBUG [PEWorker-2 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-24T16:45:19,922 DEBUG [PEWorker-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=e335232f7491,42963,1732466719025, seqNum=-1] 2024-11-24T16:45:19,923 DEBUG [PEWorker-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-24T16:45:19,924 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-8-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52111, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-24T16:45:19,929 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 673 msec 2024-11-24T16:45:19,929 INFO [master/e335232f7491:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1732466719929, completionTime=-1 2024-11-24T16:45:19,929 INFO [master/e335232f7491:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=2; waited=0ms, expected min=2 server(s), max=2 server(s), master is running 2024-11-24T16:45:19,929 DEBUG [master/e335232f7491:0:becomeActiveMaster {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-24T16:45:19,931 INFO [master/e335232f7491:0:becomeActiveMaster {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-11-24T16:45:19,931 INFO [master/e335232f7491:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732466779931 2024-11-24T16:45:19,931 INFO [master/e335232f7491:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732466839931 2024-11-24T16:45:19,932 INFO [master/e335232f7491:0:becomeActiveMaster {}] assignment.AssignmentManager(1783): Joined the cluster in 2 msec 2024-11-24T16:45:19,932 INFO [master/e335232f7491:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=e335232f7491,36357,1732466718936-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-24T16:45:19,932 INFO [master/e335232f7491:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=e335232f7491,36357,1732466718936-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-24T16:45:19,932 INFO [master/e335232f7491:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=e335232f7491,36357,1732466718936-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-24T16:45:19,932 INFO [master/e335232f7491:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-e335232f7491:36357, period=300000, unit=MILLISECONDS is enabled. 2024-11-24T16:45:19,932 INFO [master/e335232f7491:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2024-11-24T16:45:19,932 INFO [master/e335232f7491:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled. 2024-11-24T16:45:19,935 DEBUG [master/e335232f7491:0.Chore.1 {}] janitor.CatalogJanitor(180): 2024-11-24T16:45:19,937 INFO [master/e335232f7491:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 0.866sec 2024-11-24T16:45:19,937 INFO [master/e335232f7491:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2024-11-24T16:45:19,937 INFO [master/e335232f7491:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2024-11-24T16:45:19,937 INFO [master/e335232f7491:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2024-11-24T16:45:19,937 INFO [master/e335232f7491:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2024-11-24T16:45:19,937 INFO [master/e335232f7491:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2024-11-24T16:45:19,937 INFO [master/e335232f7491:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=e335232f7491,36357,1732466718936-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-24T16:45:19,937 INFO [master/e335232f7491:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=e335232f7491,36357,1732466718936-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2024-11-24T16:45:19,938 DEBUG [master/e335232f7491:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds 2024-11-24T16:45:19,938 INFO [master/e335232f7491:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2024-11-24T16:45:19,938 INFO [master/e335232f7491:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=e335232f7491,36357,1732466718936-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-24T16:45:19,949 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1fc06048, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T16:45:19,949 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request e335232f7491,36357,-1 for getting cluster id 2024-11-24T16:45:19,950 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-24T16:45:19,951 DEBUG [HMaster-EventLoopGroup-6-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '8b6ab050-aca9-4038-b19c-0b3323afc835' 2024-11-24T16:45:19,952 DEBUG [RPCClient-NioEventLoopGroup-5-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-24T16:45:19,952 DEBUG [RPCClient-NioEventLoopGroup-5-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "8b6ab050-aca9-4038-b19c-0b3323afc835" 2024-11-24T16:45:19,952 DEBUG [RPCClient-NioEventLoopGroup-5-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3eefa14a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T16:45:19,952 DEBUG [RPCClient-NioEventLoopGroup-5-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [e335232f7491,36357,-1] 2024-11-24T16:45:19,952 DEBUG [RPCClient-NioEventLoopGroup-5-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-24T16:45:19,953 DEBUG [RPCClient-NioEventLoopGroup-5-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T16:45:19,954 INFO [HMaster-EventLoopGroup-6-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44772, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-24T16:45:19,956 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@556fb9de, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T16:45:19,956 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-24T16:45:19,957 DEBUG [RPCClient-NioEventLoopGroup-5-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=e335232f7491,42963,1732466719025, seqNum=-1] 2024-11-24T16:45:19,958 DEBUG [RPCClient-NioEventLoopGroup-5-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-24T16:45:19,960 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-8-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49780, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-24T16:45:19,962 INFO [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=e335232f7491,36357,1732466718936 2024-11-24T16:45:19,962 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-24T16:45:19,963 DEBUG [RPCClient-NioEventLoopGroup-5-6 {}] client.AsyncConnectionImpl(321): The fetched master address is e335232f7491,36357,1732466718936 2024-11-24T16:45:19,963 DEBUG [RPCClient-NioEventLoopGroup-5-6 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@274f6e1e 2024-11-24T16:45:19,964 DEBUG [RPCClient-NioEventLoopGroup-5-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-24T16:45:19,965 INFO [HMaster-EventLoopGroup-6-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44778, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-24T16:45:19,966 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36357 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'testBlockEvictionOnGracefulStop1732466719962', {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-24T16:45:19,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36357 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=testBlockEvictionOnGracefulStop1732466719962 2024-11-24T16:45:19,969 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=testBlockEvictionOnGracefulStop1732466719962 execute state=CREATE_TABLE_PRE_OPERATION 2024-11-24T16:45:19,969 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:45:19,969 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36357 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "testBlockEvictionOnGracefulStop1732466719962" procId is: 4 2024-11-24T16:45:19,970 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=testBlockEvictionOnGracefulStop1732466719962 execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-24T16:45:19,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36357 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-24T16:45:19,982 INFO [RegionOpenAndInit-testBlockEvictionOnGracefulStop1732466719962-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => edbfb0487dd5f5fb1ea5f080d86356e0, NAME => 'testBlockEvictionOnGracefulStop1732466719962,,1732466719966.edbfb0487dd5f5fb1ea5f080d86356e0.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testBlockEvictionOnGracefulStop1732466719962', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9797cf4c-7cf0-a873-8ba7-f30267b4a2f7/39f4fdd4-f188-1d18-4286-5b0c4dc27cad 2024-11-24T16:45:19,990 DEBUG [RegionOpenAndInit-testBlockEvictionOnGracefulStop1732466719962-pool-0 {}] regionserver.HRegion(898): Instantiated testBlockEvictionOnGracefulStop1732466719962,,1732466719966.edbfb0487dd5f5fb1ea5f080d86356e0.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T16:45:19,990 DEBUG [RegionOpenAndInit-testBlockEvictionOnGracefulStop1732466719962-pool-0 {}] regionserver.HRegion(1722): Closing edbfb0487dd5f5fb1ea5f080d86356e0, disabling compactions & flushes 2024-11-24T16:45:19,990 INFO [RegionOpenAndInit-testBlockEvictionOnGracefulStop1732466719962-pool-0 {}] regionserver.HRegion(1755): Closing region testBlockEvictionOnGracefulStop1732466719962,,1732466719966.edbfb0487dd5f5fb1ea5f080d86356e0. 2024-11-24T16:45:19,990 DEBUG [RegionOpenAndInit-testBlockEvictionOnGracefulStop1732466719962-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on testBlockEvictionOnGracefulStop1732466719962,,1732466719966.edbfb0487dd5f5fb1ea5f080d86356e0. 2024-11-24T16:45:19,990 DEBUG [RegionOpenAndInit-testBlockEvictionOnGracefulStop1732466719962-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on testBlockEvictionOnGracefulStop1732466719962,,1732466719966.edbfb0487dd5f5fb1ea5f080d86356e0. after waiting 0 ms 2024-11-24T16:45:19,990 DEBUG [RegionOpenAndInit-testBlockEvictionOnGracefulStop1732466719962-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region testBlockEvictionOnGracefulStop1732466719962,,1732466719966.edbfb0487dd5f5fb1ea5f080d86356e0. 2024-11-24T16:45:19,990 INFO [RegionOpenAndInit-testBlockEvictionOnGracefulStop1732466719962-pool-0 {}] regionserver.HRegion(1973): Closed testBlockEvictionOnGracefulStop1732466719962,,1732466719966.edbfb0487dd5f5fb1ea5f080d86356e0. 2024-11-24T16:45:19,990 DEBUG [RegionOpenAndInit-testBlockEvictionOnGracefulStop1732466719962-pool-0 {}] regionserver.HRegion(1676): Region close journal for edbfb0487dd5f5fb1ea5f080d86356e0: Waiting for close lock at 1732466719990Disabling compacts and flushes for region at 1732466719990Disabling writes for close at 1732466719990Writing region close event to WAL at 1732466719990Closed at 1732466719990 2024-11-24T16:45:19,991 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=testBlockEvictionOnGracefulStop1732466719962 execute state=CREATE_TABLE_ADD_TO_META 2024-11-24T16:45:19,992 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testBlockEvictionOnGracefulStop1732466719962,,1732466719966.edbfb0487dd5f5fb1ea5f080d86356e0.","families":{"info":[{"qualifier":"regioninfo","vlen":78,"tag":[],"timestamp":"1732466719992"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732466719992"}]},"ts":"1732466719992"} 2024-11-24T16:45:19,994 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-24T16:45:19,995 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=testBlockEvictionOnGracefulStop1732466719962 execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-24T16:45:19,996 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testBlockEvictionOnGracefulStop1732466719962","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732466719995"}]},"ts":"1732466719995"} 2024-11-24T16:45:19,998 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=testBlockEvictionOnGracefulStop1732466719962, state=ENABLING in hbase:meta 2024-11-24T16:45:19,998 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(204): Hosts are {e335232f7491=0} racks are {/default-rack=0} 2024-11-24T16:45:19,998 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-11-24T16:45:19,998 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-11-24T16:45:19,998 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-11-24T16:45:19,998 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-11-24T16:45:19,998 INFO [PEWorker-3 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-11-24T16:45:19,998 INFO [PEWorker-3 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-11-24T16:45:19,999 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-11-24T16:45:19,999 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1732466719962, region=edbfb0487dd5f5fb1ea5f080d86356e0, ASSIGN}] 2024-11-24T16:45:20,000 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1732466719962, region=edbfb0487dd5f5fb1ea5f080d86356e0, ASSIGN 2024-11-24T16:45:20,001 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1732466719962, region=edbfb0487dd5f5fb1ea5f080d86356e0, ASSIGN; state=OFFLINE, location=e335232f7491,42963,1732466719025; forceNewPlan=false, retain=false 2024-11-24T16:45:20,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36357 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-24T16:45:20,151 INFO [e335232f7491:36357 {}] balancer.BaseLoadBalancer(388): Reassigned 1 regions. 1 retained the pre-restart assignment. 2024-11-24T16:45:20,152 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=edbfb0487dd5f5fb1ea5f080d86356e0, regionState=OPENING, regionLocation=e335232f7491,42963,1732466719025 2024-11-24T16:45:20,155 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-8-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1732466719962, region=edbfb0487dd5f5fb1ea5f080d86356e0, ASSIGN because future has completed 2024-11-24T16:45:20,155 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure edbfb0487dd5f5fb1ea5f080d86356e0, server=e335232f7491,42963,1732466719025}] 2024-11-24T16:45:20,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36357 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-24T16:45:20,310 INFO [RS_OPEN_REGION-regionserver/e335232f7491:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(132): Open testBlockEvictionOnGracefulStop1732466719962,,1732466719966.edbfb0487dd5f5fb1ea5f080d86356e0. 2024-11-24T16:45:20,310 DEBUG [RS_OPEN_REGION-regionserver/e335232f7491:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7752): Opening region: {ENCODED => edbfb0487dd5f5fb1ea5f080d86356e0, NAME => 'testBlockEvictionOnGracefulStop1732466719962,,1732466719966.edbfb0487dd5f5fb1ea5f080d86356e0.', STARTKEY => '', ENDKEY => ''} 2024-11-24T16:45:20,311 DEBUG [RS_OPEN_REGION-regionserver/e335232f7491:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testBlockEvictionOnGracefulStop1732466719962 edbfb0487dd5f5fb1ea5f080d86356e0 2024-11-24T16:45:20,311 DEBUG [RS_OPEN_REGION-regionserver/e335232f7491:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(898): Instantiated testBlockEvictionOnGracefulStop1732466719962,,1732466719966.edbfb0487dd5f5fb1ea5f080d86356e0.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T16:45:20,311 DEBUG [RS_OPEN_REGION-regionserver/e335232f7491:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7794): checking encryption for edbfb0487dd5f5fb1ea5f080d86356e0 2024-11-24T16:45:20,311 DEBUG [RS_OPEN_REGION-regionserver/e335232f7491:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7797): checking classloading for edbfb0487dd5f5fb1ea5f080d86356e0 2024-11-24T16:45:20,312 INFO [StoreOpener-edbfb0487dd5f5fb1ea5f080d86356e0-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family family of region edbfb0487dd5f5fb1ea5f080d86356e0 2024-11-24T16:45:20,316 INFO [StoreOpener-edbfb0487dd5f5fb1ea5f080d86356e0-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region edbfb0487dd5f5fb1ea5f080d86356e0 columnFamilyName family 2024-11-24T16:45:20,316 DEBUG [StoreOpener-edbfb0487dd5f5fb1ea5f080d86356e0-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:45:20,317 INFO [StoreOpener-edbfb0487dd5f5fb1ea5f080d86356e0-1 {}] regionserver.HStore(327): Store=edbfb0487dd5f5fb1ea5f080d86356e0/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T16:45:20,317 DEBUG [RS_OPEN_REGION-regionserver/e335232f7491:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1038): replaying wal for edbfb0487dd5f5fb1ea5f080d86356e0 2024-11-24T16:45:20,318 DEBUG [RS_OPEN_REGION-regionserver/e335232f7491:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9797cf4c-7cf0-a873-8ba7-f30267b4a2f7/39f4fdd4-f188-1d18-4286-5b0c4dc27cad/data/default/testBlockEvictionOnGracefulStop1732466719962/edbfb0487dd5f5fb1ea5f080d86356e0 2024-11-24T16:45:20,318 DEBUG [RS_OPEN_REGION-regionserver/e335232f7491:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9797cf4c-7cf0-a873-8ba7-f30267b4a2f7/39f4fdd4-f188-1d18-4286-5b0c4dc27cad/data/default/testBlockEvictionOnGracefulStop1732466719962/edbfb0487dd5f5fb1ea5f080d86356e0 2024-11-24T16:45:20,318 DEBUG [RS_OPEN_REGION-regionserver/e335232f7491:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1048): stopping wal replay for edbfb0487dd5f5fb1ea5f080d86356e0 2024-11-24T16:45:20,318 DEBUG [RS_OPEN_REGION-regionserver/e335232f7491:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1060): Cleaning up temporary data for edbfb0487dd5f5fb1ea5f080d86356e0 2024-11-24T16:45:20,319 DEBUG [RS_OPEN_REGION-regionserver/e335232f7491:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1093): writing seq id for edbfb0487dd5f5fb1ea5f080d86356e0 2024-11-24T16:45:20,326 DEBUG [RS_OPEN_REGION-regionserver/e335232f7491:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9797cf4c-7cf0-a873-8ba7-f30267b4a2f7/39f4fdd4-f188-1d18-4286-5b0c4dc27cad/data/default/testBlockEvictionOnGracefulStop1732466719962/edbfb0487dd5f5fb1ea5f080d86356e0/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-24T16:45:20,327 INFO [RS_OPEN_REGION-regionserver/e335232f7491:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1114): Opened edbfb0487dd5f5fb1ea5f080d86356e0; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=72665631, jitterRate=0.08280228078365326}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-24T16:45:20,327 DEBUG [RS_OPEN_REGION-regionserver/e335232f7491:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1122): Running coprocessor post-open hooks for edbfb0487dd5f5fb1ea5f080d86356e0 2024-11-24T16:45:20,328 DEBUG [RS_OPEN_REGION-regionserver/e335232f7491:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1006): Region open journal for edbfb0487dd5f5fb1ea5f080d86356e0: Running coprocessor pre-open hook at 1732466720311Writing region info on filesystem at 1732466720311Initializing all the Stores at 1732466720311Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732466720311Cleaning up temporary data from old regions at 1732466720318 (+7 ms)Running coprocessor post-open hooks at 1732466720327 (+9 ms)Region opened successfully at 1732466720328 (+1 ms) 2024-11-24T16:45:20,329 INFO [RS_OPEN_REGION-regionserver/e335232f7491:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2236): Post open deploy tasks for testBlockEvictionOnGracefulStop1732466719962,,1732466719966.edbfb0487dd5f5fb1ea5f080d86356e0., pid=6, masterSystemTime=1732466720307 2024-11-24T16:45:20,331 DEBUG [RS_OPEN_REGION-regionserver/e335232f7491:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2266): Finished post open deploy task for testBlockEvictionOnGracefulStop1732466719962,,1732466719966.edbfb0487dd5f5fb1ea5f080d86356e0. 2024-11-24T16:45:20,331 INFO [RS_OPEN_REGION-regionserver/e335232f7491:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(153): Opened testBlockEvictionOnGracefulStop1732466719962,,1732466719966.edbfb0487dd5f5fb1ea5f080d86356e0. 2024-11-24T16:45:20,332 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=edbfb0487dd5f5fb1ea5f080d86356e0, regionState=OPEN, openSeqNum=2, regionLocation=e335232f7491,42963,1732466719025 2024-11-24T16:45:20,335 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-8-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure edbfb0487dd5f5fb1ea5f080d86356e0, server=e335232f7491,42963,1732466719025 because future has completed 2024-11-24T16:45:20,338 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=6, resume processing ppid=5 2024-11-24T16:45:20,338 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure edbfb0487dd5f5fb1ea5f080d86356e0, server=e335232f7491,42963,1732466719025 in 181 msec 2024-11-24T16:45:20,340 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=5, resume processing ppid=4 2024-11-24T16:45:20,340 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1732466719962, region=edbfb0487dd5f5fb1ea5f080d86356e0, ASSIGN in 339 msec 2024-11-24T16:45:20,341 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=testBlockEvictionOnGracefulStop1732466719962 execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-24T16:45:20,341 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testBlockEvictionOnGracefulStop1732466719962","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732466720341"}]},"ts":"1732466720341"} 2024-11-24T16:45:20,343 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=testBlockEvictionOnGracefulStop1732466719962, state=ENABLED in hbase:meta 2024-11-24T16:45:20,344 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=testBlockEvictionOnGracefulStop1732466719962 execute state=CREATE_TABLE_POST_OPERATION 2024-11-24T16:45:20,346 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=testBlockEvictionOnGracefulStop1732466719962 in 377 msec 2024-11-24T16:45:20,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36357 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-24T16:45:20,597 INFO [RPCClient-NioEventLoopGroup-5-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:testBlockEvictionOnGracefulStop1732466719962 completed 2024-11-24T16:45:20,597 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table testBlockEvictionOnGracefulStop1732466719962 get assigned. Timeout = 60000ms 2024-11-24T16:45:20,597 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-24T16:45:20,601 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table testBlockEvictionOnGracefulStop1732466719962 assigned to meta. Checking AM states. 2024-11-24T16:45:20,601 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-24T16:45:20,601 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table testBlockEvictionOnGracefulStop1732466719962 assigned. 2024-11-24T16:45:20,604 DEBUG [RPCClient-NioEventLoopGroup-5-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testBlockEvictionOnGracefulStop1732466719962', row='row1', locateType=CURRENT is [region=testBlockEvictionOnGracefulStop1732466719962,,1732466719966.edbfb0487dd5f5fb1ea5f080d86356e0., hostname=e335232f7491,42963,1732466719025, seqNum=2] 2024-11-24T16:45:20,609 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing edbfb0487dd5f5fb1ea5f080d86356e0 1/1 column families, dataSize=78 B heapSize=480 B 2024-11-24T16:45:20,619 DEBUG [Time-limited test {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 1ebd1eb39df242bbae9f0d38b310e39f 2024-11-24T16:45:20,632 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9797cf4c-7cf0-a873-8ba7-f30267b4a2f7/39f4fdd4-f188-1d18-4286-5b0c4dc27cad/data/default/testBlockEvictionOnGracefulStop1732466719962/edbfb0487dd5f5fb1ea5f080d86356e0/.tmp/family/1ebd1eb39df242bbae9f0d38b310e39f is 43, key is row1/family:qf1/1/Put/seqid=0 2024-11-24T16:45:20,632 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=78 B at sequenceid=6 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9797cf4c-7cf0-a873-8ba7-f30267b4a2f7/39f4fdd4-f188-1d18-4286-5b0c4dc27cad/data/default/testBlockEvictionOnGracefulStop1732466719962/edbfb0487dd5f5fb1ea5f080d86356e0/.tmp/family/1ebd1eb39df242bbae9f0d38b310e39f 2024-11-24T16:45:20,635 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9797cf4c-7cf0-a873-8ba7-f30267b4a2f7/39f4fdd4-f188-1d18-4286-5b0c4dc27cad/data/default/testBlockEvictionOnGracefulStop1732466719962/edbfb0487dd5f5fb1ea5f080d86356e0/.tmp/family/1ebd1eb39df242bbae9f0d38b310e39f as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9797cf4c-7cf0-a873-8ba7-f30267b4a2f7/39f4fdd4-f188-1d18-4286-5b0c4dc27cad/data/default/testBlockEvictionOnGracefulStop1732466719962/edbfb0487dd5f5fb1ea5f080d86356e0/family/1ebd1eb39df242bbae9f0d38b310e39f 2024-11-24T16:45:20,636 DEBUG [Time-limited test {}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9797cf4c-7cf0-a873-8ba7-f30267b4a2f7/39f4fdd4-f188-1d18-4286-5b0c4dc27cad/data/default/testBlockEvictionOnGracefulStop1732466719962/edbfb0487dd5f5fb1ea5f080d86356e0/family/1ebd1eb39df242bbae9f0d38b310e39f, delay=992 ms 2024-11-24T16:45:20,637 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9797cf4c-7cf0-a873-8ba7-f30267b4a2f7/39f4fdd4-f188-1d18-4286-5b0c4dc27cad/data/default/testBlockEvictionOnGracefulStop1732466719962/edbfb0487dd5f5fb1ea5f080d86356e0/family/1ebd1eb39df242bbae9f0d38b310e39f, entries=2, sequenceid=6, filesize=4.9 K 2024-11-24T16:45:20,638 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~78 B/78, heapSize ~464 B/464, currentSize=0 B/0 for edbfb0487dd5f5fb1ea5f080d86356e0 in 29ms, sequenceid=6, compaction requested=false 2024-11-24T16:45:20,638 DEBUG [Time-limited test {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'testBlockEvictionOnGracefulStop1732466719962' 2024-11-24T16:45:20,639 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for edbfb0487dd5f5fb1ea5f080d86356e0: 2024-11-24T16:45:21,292 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-24T16:45:21,629 DEBUG [hfile-prefetch-1732466718378 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 1ebd1eb39df242bbae9f0d38b310e39f_0. 2024-11-24T16:45:21,630 DEBUG [hfile-prefetch-1732466718378 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 1ebd1eb39df242bbae9f0d38b310e39f_117. 2024-11-24T16:45:21,630 DEBUG [hfile-prefetch-1732466718378 {}] bucket.BucketCache(2281): Notifying caching completed for file file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9797cf4c-7cf0-a873-8ba7-f30267b4a2f7/39f4fdd4-f188-1d18-4286-5b0c4dc27cad/data/default/testBlockEvictionOnGracefulStop1732466719962/edbfb0487dd5f5fb1ea5f080d86356e0/family/1ebd1eb39df242bbae9f0d38b310e39f, with total blocks 2, and data blocks 1 2024-11-24T16:45:21,630 DEBUG [hfile-prefetch-1732466718378 {}] bucket.BucketCache(2285): iterating over 0 entries in the backing map 2024-11-24T16:45:21,630 DEBUG [hfile-prefetch-1732466718378 {}] bucket.BucketCache(2308): Prefetch executor completed for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9797cf4c-7cf0-a873-8ba7-f30267b4a2f7/39f4fdd4-f188-1d18-4286-5b0c4dc27cad/data/default/testBlockEvictionOnGracefulStop1732466719962/edbfb0487dd5f5fb1ea5f080d86356e0/family/1ebd1eb39df242bbae9f0d38b310e39f, but only 0 data blocks were cached. Total data blocks for file: 1. Checking for blocks pending cache in cache writer queue. 2024-11-24T16:45:21,630 DEBUG [hfile-prefetch-1732466718378 {}] bucket.BucketCache(2318): There are still blocks pending caching for file file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9797cf4c-7cf0-a873-8ba7-f30267b4a2f7/39f4fdd4-f188-1d18-4286-5b0c4dc27cad/data/default/testBlockEvictionOnGracefulStop1732466719962/edbfb0487dd5f5fb1ea5f080d86356e0/family/1ebd1eb39df242bbae9f0d38b310e39f. Will sleep 100ms and try the verification again. 2024-11-24T16:45:21,637 DEBUG [bucket-cache-persister {}] bucket.BucketCachePersister(47): Cache is inconsistent, persisting to disk 2024-11-24T16:45:21,638 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1487): Thread bucket-cache-persister started persisting bucket cache to file 2024-11-24T16:45:21,638 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1494): Persist in new chunked persistence format. 2024-11-24T16:45:21,638 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1673): persistToFile: before persisting backing map size: 1, fullycachedFiles size: 0, chunkSize: 10000 2024-11-24T16:45:21,639 INFO [Time-limited test {}] hbase.SingleProcessHBaseCluster(504): Stopping Thread[RS:1;e335232f7491:42963,5,FailOnTimeoutGroup] 2024-11-24T16:45:21,640 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server 'e335232f7491,42963,1732466719025' ***** 2024-11-24T16:45:21,640 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Stopping rs 1 2024-11-24T16:45:21,640 INFO [RS:1;e335232f7491:42963 {}] bucket.BucketCache(1738): Disabling cache 2024-11-24T16:45:21,640 INFO [RS:1;e335232f7491:42963 {}] bucket.BucketCache(1762): Shutdown bucket cache: IO persistent=true; path to write=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/98608d84-f0c0-ab20-75c0-76a28c1904b5/bucket.persistence 2024-11-24T16:45:21,640 INFO [Time-limited test-BucketCacheWriter-1 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-1 exiting, cacheEnabled=false 2024-11-24T16:45:21,640 INFO [Time-limited test-BucketCacheWriter-0 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-0 exiting, cacheEnabled=false 2024-11-24T16:45:21,640 INFO [Time-limited test-BucketCacheWriter-2 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-2 exiting, cacheEnabled=false 2024-11-24T16:45:21,640 INFO [RS:1;e335232f7491:42963 {}] bucket.BucketCache(1768): Shutting down cache persister thread. 2024-11-24T16:45:21,641 ERROR [bucket-cache-persister {}] bucket.PersistentIOEngine(78): Calculating checksum failed, because of java.io.InterruptedIOException: java.lang.InterruptedException at org.apache.hadoop.util.Shell.runCommand(Shell.java:1071) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.util.Shell.run(Shell.java:959) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.util.Shell$ShellCommandExecutor.execute(Shell.java:1282) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.io.hfile.bucket.PersistentIOEngine.getFileSize(PersistentIOEngine.java:94) ~[classes/:?] at org.apache.hadoop.hbase.io.hfile.bucket.PersistentIOEngine.calculateChecksum(PersistentIOEngine.java:71) ~[classes/:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketProtoUtils.toPB(BucketProtoUtils.java:61) ~[classes/:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketProtoUtils.serializeAsPB(BucketProtoUtils.java:75) ~[classes/:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCache.persistChunkedBackingMap(BucketCache.java:1678) ~[classes/:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCache.persistToFile(BucketCache.java:1496) ~[classes/:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCachePersister.run(BucketCachePersister.java:48) ~[classes/:?] Caused by: java.lang.InterruptedException at java.lang.Object.wait(Native Method) ~[?:?] at java.lang.Object.wait(Object.java:338) ~[?:?] at java.lang.ProcessImpl.waitFor(ProcessImpl.java:434) ~[?:?] at org.apache.hadoop.util.Shell.runCommand(Shell.java:1061) ~[hadoop-common-3.4.1.jar:?] ... 9 more 2024-11-24T16:45:21,645 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1680): persistToFile: after persisting backing map size: 1, fullycachedFiles size: 0 2024-11-24T16:45:21,645 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1498): PersistToFile: after persisting backing map size: 1, fullycachedFiles size: 0, file name: bucket.persistence1732466721638 2024-11-24T16:45:21,645 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1509): Thread bucket-cache-persister finished persisting bucket cache to file, renaming 2024-11-24T16:45:21,645 INFO [bucket-cache-persister {}] bucket.BucketCachePersister(62): Finishing cache persister thread. 2024-11-24T16:45:21,650 DEBUG [RS:1;e335232f7491:42963 {}] bucket.BucketCache(1487): Thread RS:1;e335232f7491:42963 started persisting bucket cache to file 2024-11-24T16:45:21,650 DEBUG [RS:1;e335232f7491:42963 {}] bucket.BucketCache(1494): Persist in new chunked persistence format. 2024-11-24T16:45:21,650 DEBUG [RS:1;e335232f7491:42963 {}] bucket.BucketCache(1673): persistToFile: before persisting backing map size: 1, fullycachedFiles size: 0, chunkSize: 10000 2024-11-24T16:45:21,653 DEBUG [RS:1;e335232f7491:42963 {}] bucket.BucketCache(1680): persistToFile: after persisting backing map size: 1, fullycachedFiles size: 0 2024-11-24T16:45:21,654 DEBUG [RS:1;e335232f7491:42963 {}] bucket.BucketCache(1498): PersistToFile: after persisting backing map size: 1, fullycachedFiles size: 0, file name: bucket.persistence1732466721650 2024-11-24T16:45:21,654 DEBUG [RS:1;e335232f7491:42963 {}] bucket.BucketCache(1509): Thread RS:1;e335232f7491:42963 finished persisting bucket cache to file, renaming 2024-11-24T16:45:21,654 INFO [RS:1;e335232f7491:42963 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-24T16:45:21,654 INFO [RS:1;e335232f7491:42963 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-24T16:45:21,654 INFO [RS:1;e335232f7491:42963 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-24T16:45:21,654 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-24T16:45:21,654 INFO [RS:1;e335232f7491:42963 {}] regionserver.HRegionServer(3091): Received CLOSE for edbfb0487dd5f5fb1ea5f080d86356e0 2024-11-24T16:45:21,654 INFO [RS:1;e335232f7491:42963 {}] regionserver.HRegionServer(959): stopping server e335232f7491,42963,1732466719025 2024-11-24T16:45:21,654 INFO [RS:1;e335232f7491:42963 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-24T16:45:21,654 INFO [RS:1;e335232f7491:42963 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:1;e335232f7491:42963. 2024-11-24T16:45:21,654 INFO [RS_CLOSE_REGION-regionserver/e335232f7491:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1662): Closing region testBlockEvictionOnGracefulStop1732466719962,,1732466719966.edbfb0487dd5f5fb1ea5f080d86356e0. during a graceful stop, and cache persistence is on, so setting evict on close to false. 2024-11-24T16:45:21,655 DEBUG [RS:1;e335232f7491:42963 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-24T16:45:21,655 DEBUG [RS_CLOSE_REGION-regionserver/e335232f7491:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing edbfb0487dd5f5fb1ea5f080d86356e0, disabling compactions & flushes 2024-11-24T16:45:21,655 DEBUG [RS:1;e335232f7491:42963 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T16:45:21,655 INFO [RS_CLOSE_REGION-regionserver/e335232f7491:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testBlockEvictionOnGracefulStop1732466719962,,1732466719966.edbfb0487dd5f5fb1ea5f080d86356e0. 2024-11-24T16:45:21,655 DEBUG [RS_CLOSE_REGION-regionserver/e335232f7491:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testBlockEvictionOnGracefulStop1732466719962,,1732466719966.edbfb0487dd5f5fb1ea5f080d86356e0. 2024-11-24T16:45:21,655 INFO [RS:1;e335232f7491:42963 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-24T16:45:21,655 INFO [RS:1;e335232f7491:42963 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-24T16:45:21,655 DEBUG [RS_CLOSE_REGION-regionserver/e335232f7491:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testBlockEvictionOnGracefulStop1732466719962,,1732466719966.edbfb0487dd5f5fb1ea5f080d86356e0. after waiting 0 ms 2024-11-24T16:45:21,655 INFO [RS:1;e335232f7491:42963 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-24T16:45:21,655 DEBUG [RS_CLOSE_REGION-regionserver/e335232f7491:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testBlockEvictionOnGracefulStop1732466719962,,1732466719966.edbfb0487dd5f5fb1ea5f080d86356e0. 2024-11-24T16:45:21,655 INFO [RS:1;e335232f7491:42963 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-11-24T16:45:21,655 INFO [RS:1;e335232f7491:42963 {}] regionserver.HRegionServer(1321): Waiting on 2 regions to close 2024-11-24T16:45:21,655 DEBUG [RS:1;e335232f7491:42963 {}] regionserver.HRegionServer(1325): Online Regions={1588230740=hbase:meta,,1.1588230740, edbfb0487dd5f5fb1ea5f080d86356e0=testBlockEvictionOnGracefulStop1732466719962,,1732466719966.edbfb0487dd5f5fb1ea5f080d86356e0.} 2024-11-24T16:45:21,655 DEBUG [RS:1;e335232f7491:42963 {}] regionserver.HRegionServer(1351): Waiting on 1588230740, edbfb0487dd5f5fb1ea5f080d86356e0 2024-11-24T16:45:21,655 INFO [RS_CLOSE_META-regionserver/e335232f7491:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1662): Closing region hbase:meta,,1.1588230740 during a graceful stop, and cache persistence is on, so setting evict on close to false. 2024-11-24T16:45:21,655 DEBUG [RS_CLOSE_META-regionserver/e335232f7491:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-24T16:45:21,655 INFO [RS_CLOSE_META-regionserver/e335232f7491:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-24T16:45:21,656 DEBUG [RS_CLOSE_META-regionserver/e335232f7491:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-24T16:45:21,656 DEBUG [RS_CLOSE_META-regionserver/e335232f7491:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-24T16:45:21,656 DEBUG [RS_CLOSE_META-regionserver/e335232f7491:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-24T16:45:21,656 INFO [RS_CLOSE_META-regionserver/e335232f7491:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=1.75 KB heapSize=3.78 KB 2024-11-24T16:45:21,656 DEBUG [StoreFileCloser-edbfb0487dd5f5fb1ea5f080d86356e0-family-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9797cf4c-7cf0-a873-8ba7-f30267b4a2f7/39f4fdd4-f188-1d18-4286-5b0c4dc27cad/data/default/testBlockEvictionOnGracefulStop1732466719962/edbfb0487dd5f5fb1ea5f080d86356e0/family/1ebd1eb39df242bbae9f0d38b310e39f 2024-11-24T16:45:21,656 WARN [hfile-prefetch-1732466718378 {}] hfile.HFilePreadReader$1(158): Prefetch path=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9797cf4c-7cf0-a873-8ba7-f30267b4a2f7/39f4fdd4-f188-1d18-4286-5b0c4dc27cad/data/default/testBlockEvictionOnGracefulStop1732466719962/edbfb0487dd5f5fb1ea5f080d86356e0/family/1ebd1eb39df242bbae9f0d38b310e39f, offset=158, end=158 java.lang.RuntimeException: java.lang.InterruptedException: sleep interrupted at org.apache.hadoop.hbase.io.hfile.bucket.BucketCache.notifyFileCachingCompleted(BucketCache.java:2337) ~[classes/:?] at org.apache.hadoop.hbase.io.hfile.CombinedBlockCache.notifyFileCachingCompleted(CombinedBlockCache.java:469) ~[classes/:?] at org.apache.hadoop.hbase.io.hfile.HFilePreadReader$1.run(HFilePreadReader.java:148) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.lambda$tracedRunnable$2(TraceUtil.java:155) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:539) ~[?:?] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:304) ~[?:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCache.notifyFileCachingCompleted(BucketCache.java:2320) ~[classes/:?] ... 9 more 2024-11-24T16:45:21,657 DEBUG [hfile-prefetch-1732466718378 {}] hfile.PrefetchExecutor(121): Prefetch completed for 1ebd1eb39df242bbae9f0d38b310e39f 2024-11-24T16:45:21,661 DEBUG [RS_CLOSE_REGION-regionserver/e335232f7491:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9797cf4c-7cf0-a873-8ba7-f30267b4a2f7/39f4fdd4-f188-1d18-4286-5b0c4dc27cad/data/default/testBlockEvictionOnGracefulStop1732466719962/edbfb0487dd5f5fb1ea5f080d86356e0/recovered.edits/9.seqid, newMaxSeqId=9, maxSeqId=1 2024-11-24T16:45:21,661 INFO [RS_CLOSE_REGION-regionserver/e335232f7491:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testBlockEvictionOnGracefulStop1732466719962,,1732466719966.edbfb0487dd5f5fb1ea5f080d86356e0. 2024-11-24T16:45:21,661 DEBUG [RS_CLOSE_REGION-regionserver/e335232f7491:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for edbfb0487dd5f5fb1ea5f080d86356e0: Waiting for close lock at 1732466721654Running coprocessor pre-close hooks at 1732466721655 (+1 ms)Disabling compacts and flushes for region at 1732466721655Disabling writes for close at 1732466721655Writing region close event to WAL at 1732466721656 (+1 ms)Running coprocessor post-close hooks at 1732466721661 (+5 ms)Closed at 1732466721661 2024-11-24T16:45:21,661 DEBUG [RS_CLOSE_REGION-regionserver/e335232f7491:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testBlockEvictionOnGracefulStop1732466719962,,1732466719966.edbfb0487dd5f5fb1ea5f080d86356e0. 2024-11-24T16:45:21,667 DEBUG [RS_CLOSE_META-regionserver/e335232f7491:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 971b3bd28c36406c9e7e88e8e2172769 2024-11-24T16:45:21,681 DEBUG [RS_CLOSE_META-regionserver/e335232f7491:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9797cf4c-7cf0-a873-8ba7-f30267b4a2f7/39f4fdd4-f188-1d18-4286-5b0c4dc27cad/data/hbase/meta/1588230740/.tmp/info/971b3bd28c36406c9e7e88e8e2172769 is 209, key is testBlockEvictionOnGracefulStop1732466719962,,1732466719966.edbfb0487dd5f5fb1ea5f080d86356e0./info:regioninfo/1732466720332/Put/seqid=0 2024-11-24T16:45:21,682 INFO [RS_CLOSE_META-regionserver/e335232f7491:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.53 KB at sequenceid=11 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9797cf4c-7cf0-a873-8ba7-f30267b4a2f7/39f4fdd4-f188-1d18-4286-5b0c4dc27cad/data/hbase/meta/1588230740/.tmp/info/971b3bd28c36406c9e7e88e8e2172769 2024-11-24T16:45:21,690 DEBUG [RS_CLOSE_META-regionserver/e335232f7491:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFile$WriterFactory(308): Unable to set drop behind on c1c6018341f54195884d54ad8cd64f1d 2024-11-24T16:45:21,697 INFO [regionserver/e335232f7491:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-24T16:45:21,703 DEBUG [RS_CLOSE_META-regionserver/e335232f7491:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9797cf4c-7cf0-a873-8ba7-f30267b4a2f7/39f4fdd4-f188-1d18-4286-5b0c4dc27cad/data/hbase/meta/1588230740/.tmp/ns/c1c6018341f54195884d54ad8cd64f1d is 43, key is default/ns:d/1732466719925/Put/seqid=0 2024-11-24T16:45:21,704 INFO [RS_CLOSE_META-regionserver/e335232f7491:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=74 B at sequenceid=11 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9797cf4c-7cf0-a873-8ba7-f30267b4a2f7/39f4fdd4-f188-1d18-4286-5b0c4dc27cad/data/hbase/meta/1588230740/.tmp/ns/c1c6018341f54195884d54ad8cd64f1d 2024-11-24T16:45:21,712 DEBUG [RS_CLOSE_META-regionserver/e335232f7491:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 6ee5d4979983472db67dd250d09f8b2a 2024-11-24T16:45:21,725 DEBUG [RS_CLOSE_META-regionserver/e335232f7491:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9797cf4c-7cf0-a873-8ba7-f30267b4a2f7/39f4fdd4-f188-1d18-4286-5b0c4dc27cad/data/hbase/meta/1588230740/.tmp/table/6ee5d4979983472db67dd250d09f8b2a is 80, key is testBlockEvictionOnGracefulStop1732466719962/table:state/1732466720341/Put/seqid=0 2024-11-24T16:45:21,725 INFO [RS_CLOSE_META-regionserver/e335232f7491:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=152 B at sequenceid=11 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9797cf4c-7cf0-a873-8ba7-f30267b4a2f7/39f4fdd4-f188-1d18-4286-5b0c4dc27cad/data/hbase/meta/1588230740/.tmp/table/6ee5d4979983472db67dd250d09f8b2a 2024-11-24T16:45:21,728 DEBUG [RS_CLOSE_META-regionserver/e335232f7491:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9797cf4c-7cf0-a873-8ba7-f30267b4a2f7/39f4fdd4-f188-1d18-4286-5b0c4dc27cad/data/hbase/meta/1588230740/.tmp/info/971b3bd28c36406c9e7e88e8e2172769 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9797cf4c-7cf0-a873-8ba7-f30267b4a2f7/39f4fdd4-f188-1d18-4286-5b0c4dc27cad/data/hbase/meta/1588230740/info/971b3bd28c36406c9e7e88e8e2172769 2024-11-24T16:45:21,729 DEBUG [RS_CLOSE_META-regionserver/e335232f7491:0-0 {event_type=M_RS_CLOSE_META}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9797cf4c-7cf0-a873-8ba7-f30267b4a2f7/39f4fdd4-f188-1d18-4286-5b0c4dc27cad/data/hbase/meta/1588230740/info/971b3bd28c36406c9e7e88e8e2172769, delay=962 ms 2024-11-24T16:45:21,730 INFO [RS_CLOSE_META-regionserver/e335232f7491:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9797cf4c-7cf0-a873-8ba7-f30267b4a2f7/39f4fdd4-f188-1d18-4286-5b0c4dc27cad/data/hbase/meta/1588230740/info/971b3bd28c36406c9e7e88e8e2172769, entries=10, sequenceid=11, filesize=7.0 K 2024-11-24T16:45:21,730 DEBUG [RS_CLOSE_META-regionserver/e335232f7491:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9797cf4c-7cf0-a873-8ba7-f30267b4a2f7/39f4fdd4-f188-1d18-4286-5b0c4dc27cad/data/hbase/meta/1588230740/.tmp/ns/c1c6018341f54195884d54ad8cd64f1d as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9797cf4c-7cf0-a873-8ba7-f30267b4a2f7/39f4fdd4-f188-1d18-4286-5b0c4dc27cad/data/hbase/meta/1588230740/ns/c1c6018341f54195884d54ad8cd64f1d 2024-11-24T16:45:21,731 DEBUG [RS_CLOSE_META-regionserver/e335232f7491:0-0 {event_type=M_RS_CLOSE_META}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9797cf4c-7cf0-a873-8ba7-f30267b4a2f7/39f4fdd4-f188-1d18-4286-5b0c4dc27cad/data/hbase/meta/1588230740/ns/c1c6018341f54195884d54ad8cd64f1d, delay=916 ms 2024-11-24T16:45:21,732 INFO [RS_CLOSE_META-regionserver/e335232f7491:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9797cf4c-7cf0-a873-8ba7-f30267b4a2f7/39f4fdd4-f188-1d18-4286-5b0c4dc27cad/data/hbase/meta/1588230740/ns/c1c6018341f54195884d54ad8cd64f1d, entries=2, sequenceid=11, filesize=5.0 K 2024-11-24T16:45:21,733 DEBUG [RS_CLOSE_META-regionserver/e335232f7491:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9797cf4c-7cf0-a873-8ba7-f30267b4a2f7/39f4fdd4-f188-1d18-4286-5b0c4dc27cad/data/hbase/meta/1588230740/.tmp/table/6ee5d4979983472db67dd250d09f8b2a as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9797cf4c-7cf0-a873-8ba7-f30267b4a2f7/39f4fdd4-f188-1d18-4286-5b0c4dc27cad/data/hbase/meta/1588230740/table/6ee5d4979983472db67dd250d09f8b2a 2024-11-24T16:45:21,734 DEBUG [RS_CLOSE_META-regionserver/e335232f7491:0-0 {event_type=M_RS_CLOSE_META}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9797cf4c-7cf0-a873-8ba7-f30267b4a2f7/39f4fdd4-f188-1d18-4286-5b0c4dc27cad/data/hbase/meta/1588230740/table/6ee5d4979983472db67dd250d09f8b2a, delay=990 ms 2024-11-24T16:45:21,734 INFO [RS_CLOSE_META-regionserver/e335232f7491:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9797cf4c-7cf0-a873-8ba7-f30267b4a2f7/39f4fdd4-f188-1d18-4286-5b0c4dc27cad/data/hbase/meta/1588230740/table/6ee5d4979983472db67dd250d09f8b2a, entries=2, sequenceid=11, filesize=5.3 K 2024-11-24T16:45:21,735 INFO [RS_CLOSE_META-regionserver/e335232f7491:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~1.75 KB/1797, heapSize ~3.48 KB/3568, currentSize=0 B/0 for 1588230740 in 79ms, sequenceid=11, compaction requested=false 2024-11-24T16:45:21,736 DEBUG [StoreFileCloser-1588230740-info-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9797cf4c-7cf0-a873-8ba7-f30267b4a2f7/39f4fdd4-f188-1d18-4286-5b0c4dc27cad/data/hbase/meta/1588230740/info/971b3bd28c36406c9e7e88e8e2172769 2024-11-24T16:45:21,736 DEBUG [StoreFileCloser-1588230740-ns-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9797cf4c-7cf0-a873-8ba7-f30267b4a2f7/39f4fdd4-f188-1d18-4286-5b0c4dc27cad/data/hbase/meta/1588230740/ns/c1c6018341f54195884d54ad8cd64f1d 2024-11-24T16:45:21,737 DEBUG [StoreFileCloser-1588230740-table-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9797cf4c-7cf0-a873-8ba7-f30267b4a2f7/39f4fdd4-f188-1d18-4286-5b0c4dc27cad/data/hbase/meta/1588230740/table/6ee5d4979983472db67dd250d09f8b2a 2024-11-24T16:45:21,741 DEBUG [RS_CLOSE_META-regionserver/e335232f7491:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9797cf4c-7cf0-a873-8ba7-f30267b4a2f7/39f4fdd4-f188-1d18-4286-5b0c4dc27cad/data/hbase/meta/1588230740/recovered.edits/14.seqid, newMaxSeqId=14, maxSeqId=1 2024-11-24T16:45:21,742 DEBUG [RS_CLOSE_META-regionserver/e335232f7491:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-24T16:45:21,742 INFO [RS_CLOSE_META-regionserver/e335232f7491:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-24T16:45:21,742 DEBUG [RS_CLOSE_META-regionserver/e335232f7491:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1732466721655Running coprocessor pre-close hooks at 1732466721655Disabling compacts and flushes for region at 1732466721655Disabling writes for close at 1732466721656 (+1 ms)Obtaining lock to block concurrent updates at 1732466721656Preparing flush snapshotting stores in 1588230740 at 1732466721656Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=1797, getHeapSize=3808, getOffHeapSize=0, getCellsCount=14 at 1732466721656Flushing stores of hbase:meta,,1.1588230740 at 1732466721657 (+1 ms)Flushing 1588230740/info: creating writer at 1732466721657Flushing 1588230740/info: appending metadata at 1732466721681 (+24 ms)Flushing 1588230740/info: closing flushed file at 1732466721681Flushing 1588230740/ns: creating writer at 1732466721684 (+3 ms)Flushing 1588230740/ns: appending metadata at 1732466721703 (+19 ms)Flushing 1588230740/ns: closing flushed file at 1732466721703Flushing 1588230740/table: creating writer at 1732466721706 (+3 ms)Flushing 1588230740/table: appending metadata at 1732466721724 (+18 ms)Flushing 1588230740/table: closing flushed file at 1732466721724Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@77509ada: reopening flushed file at 1732466721728 (+4 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@48c8590c: reopening flushed file at 1732466721730 (+2 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@684e2eca: reopening flushed file at 1732466721732 (+2 ms)Finished flush of dataSize ~1.75 KB/1797, heapSize ~3.48 KB/3568, currentSize=0 B/0 for 1588230740 in 79ms, sequenceid=11, compaction requested=false at 1732466721735 (+3 ms)Writing region close event to WAL at 1732466721737 (+2 ms)Running coprocessor post-close hooks at 1732466721741 (+4 ms)Closed at 1732466721742 (+1 ms) 2024-11-24T16:45:21,742 DEBUG [RS_CLOSE_META-regionserver/e335232f7491:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-11-24T16:45:21,855 INFO [RS:1;e335232f7491:42963 {}] regionserver.HRegionServer(976): stopping server e335232f7491,42963,1732466719025; all regions closed. 2024-11-24T16:45:21,858 DEBUG [RS:1;e335232f7491:42963 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9797cf4c-7cf0-a873-8ba7-f30267b4a2f7/39f4fdd4-f188-1d18-4286-5b0c4dc27cad/oldWALs 2024-11-24T16:45:21,858 INFO [RS:1;e335232f7491:42963 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL e335232f7491%2C42963%2C1732466719025.meta:.meta(num 1732466719879) 2024-11-24T16:45:21,859 DEBUG [RS:1;e335232f7491:42963 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9797cf4c-7cf0-a873-8ba7-f30267b4a2f7/39f4fdd4-f188-1d18-4286-5b0c4dc27cad/oldWALs 2024-11-24T16:45:21,859 INFO [RS:1;e335232f7491:42963 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL e335232f7491%2C42963%2C1732466719025:(num 1732466719536) 2024-11-24T16:45:21,860 DEBUG [RS:1;e335232f7491:42963 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T16:45:21,860 INFO [RS:1;e335232f7491:42963 {}] regionserver.LeaseManager(133): Closed leases 2024-11-24T16:45:21,860 INFO [RS:1;e335232f7491:42963 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-24T16:45:21,860 INFO [RS:1;e335232f7491:42963 {}] hbase.ChoreService(370): Chore service for: regionserver/e335232f7491:0 had [ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-11-24T16:45:21,860 INFO [RS:1;e335232f7491:42963 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-24T16:45:21,860 INFO [regionserver/e335232f7491:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-24T16:45:21,860 INFO [RS:1;e335232f7491:42963 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:42963 2024-11-24T16:45:21,865 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42963-0x1014ade23570002, quorum=127.0.0.1:63542, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/e335232f7491,42963,1732466719025 2024-11-24T16:45:21,865 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36357-0x1014ade23570000, quorum=127.0.0.1:63542, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-24T16:45:21,865 INFO [RS:1;e335232f7491:42963 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-24T16:45:21,865 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [e335232f7491,42963,1732466719025] 2024-11-24T16:45:21,869 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/e335232f7491,42963,1732466719025 already deleted, retry=false 2024-11-24T16:45:21,869 INFO [RegionServerTracker-0 {}] master.ServerManager(695): Processing expiration of e335232f7491,42963,1732466719025 on e335232f7491,36357,1732466718936 2024-11-24T16:45:21,874 DEBUG [RegionServerTracker-0 {}] procedure2.ProcedureExecutor(1139): Stored pid=7, state=RUNNABLE:SERVER_CRASH_START, hasLock=false; ServerCrashProcedure e335232f7491,42963,1732466719025, splitWal=true, meta=true 2024-11-24T16:45:21,876 INFO [RegionServerTracker-0 {}] assignment.AssignmentManager(1999): Scheduled ServerCrashProcedure pid=7 for e335232f7491,42963,1732466719025 (carryingMeta=true) e335232f7491,42963,1732466719025/CRASHED/regionCount=2/lock=java.util.concurrent.locks.ReentrantReadWriteLock@3246ea3a[Write locks = 1, Read locks = 0], oldState=ONLINE. 2024-11-24T16:45:21,876 INFO [PEWorker-2 {}] procedure.ServerCrashProcedure(169): Start pid=7, state=RUNNABLE:SERVER_CRASH_START, hasLock=true; ServerCrashProcedure e335232f7491,42963,1732466719025, splitWal=true, meta=true 2024-11-24T16:45:21,877 INFO [PEWorker-2 {}] procedure.ServerCrashProcedure(339): Splitting WALs pid=7, state=RUNNABLE:SERVER_CRASH_SPLIT_META_LOGS, hasLock=true; ServerCrashProcedure e335232f7491,42963,1732466719025, splitWal=true, meta=true, isMeta: true 2024-11-24T16:45:21,878 DEBUG [PEWorker-2 {}] master.MasterWalManager(329): Renamed region directory: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9797cf4c-7cf0-a873-8ba7-f30267b4a2f7/39f4fdd4-f188-1d18-4286-5b0c4dc27cad/WALs/e335232f7491,42963,1732466719025-splitting 2024-11-24T16:45:21,878 INFO [PEWorker-2 {}] master.SplitLogManager(171): file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9797cf4c-7cf0-a873-8ba7-f30267b4a2f7/39f4fdd4-f188-1d18-4286-5b0c4dc27cad/WALs/e335232f7491,42963,1732466719025-splitting dir is empty, no logs to split. 2024-11-24T16:45:21,878 INFO [PEWorker-2 {}] master.SplitWALManager(105): e335232f7491,42963,1732466719025 WAL count=0, meta=true 2024-11-24T16:45:21,880 INFO [PEWorker-2 {}] master.SplitLogManager(171): file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9797cf4c-7cf0-a873-8ba7-f30267b4a2f7/39f4fdd4-f188-1d18-4286-5b0c4dc27cad/WALs/e335232f7491,42963,1732466719025-splitting dir is empty, no logs to split. 2024-11-24T16:45:21,880 INFO [PEWorker-2 {}] master.SplitWALManager(105): e335232f7491,42963,1732466719025 WAL count=0, meta=true 2024-11-24T16:45:21,880 DEBUG [PEWorker-2 {}] procedure.ServerCrashProcedure(329): Check if e335232f7491,42963,1732466719025 WAL splitting is done? wals=0, meta=true 2024-11-24T16:45:21,881 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=8, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-11-24T16:45:21,882 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=8, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-24T16:45:21,883 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(269): Starting pid=8, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-11-24T16:45:21,968 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42963-0x1014ade23570002, quorum=127.0.0.1:63542, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-24T16:45:21,968 INFO [RS:1;e335232f7491:42963 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-24T16:45:21,969 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42963-0x1014ade23570002, quorum=127.0.0.1:63542, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-24T16:45:21,969 INFO [RS:1;e335232f7491:42963 {}] regionserver.HRegionServer(1031): Exiting; stopping=e335232f7491,42963,1732466719025; zookeeper connection closed. 2024-11-24T16:45:21,969 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@1fef876c {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@1fef876c 2024-11-24T16:45:22,033 DEBUG [e335232f7491:36357 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=1, allServersCount=1 2024-11-24T16:45:22,033 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=8 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=e335232f7491,33045,1732466718984 2024-11-24T16:45:22,035 INFO [PEWorker-4 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as e335232f7491,33045,1732466718984, state=OPENING 2024-11-24T16:45:22,036 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36357-0x1014ade23570000, quorum=127.0.0.1:63542, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-24T16:45:22,036 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33045-0x1014ade23570001, quorum=127.0.0.1:63542, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-24T16:45:22,037 DEBUG [PEWorker-4 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=8, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-24T16:45:22,037 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-24T16:45:22,037 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-24T16:45:22,037 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=9, ppid=8, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=e335232f7491,33045,1732466718984}] 2024-11-24T16:45:22,156 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/e335232f7491:0 server-side Connection retries=45 2024-11-24T16:45:22,156 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-24T16:45:22,156 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-24T16:45:22,156 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-24T16:45:22,157 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-24T16:45:22,157 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-24T16:45:22,157 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-24T16:45:22,157 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-24T16:45:22,158 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:41507 2024-11-24T16:45:22,159 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:41507 connecting to ZooKeeper ensemble=127.0.0.1:63542 2024-11-24T16:45:22,159 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-24T16:45:22,159 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-24T16:45:22,162 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:415070x0, quorum=127.0.0.1:63542, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-24T16:45:22,163 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:41507-0x1014ade23570003 connected 2024-11-24T16:45:22,163 DEBUG [Time-limited test {}] zookeeper.ZKUtil(111): regionserver:41507-0x1014ade23570003, quorum=127.0.0.1:63542, baseZNode=/hbase Set watcher on existing znode=/hbase/running 2024-11-24T16:45:22,163 DEBUG [pool-298-thread-1 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: INIT 2024-11-24T16:45:22,163 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-24T16:45:22,164 INFO [Time-limited test {}] bucket.FileIOEngine(99): Allocating cache 400 MB, on the path:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/98608d84-f0c0-ab20-75c0-76a28c1904b5/bucket.cache 2024-11-24T16:45:22,164 INFO [Time-limited test {}] bucket.BucketCache(364): Instantiating BucketCache with acceptableFactor: 0.95, minFactor: 0.85, extraFreeFactor: 0.1, singleFactor: 0.25, multiFactor: 0.5, memoryFactor: 0.25, useStrongRef: false 2024-11-24T16:45:22,164 INFO [Time-limited test {}] bucket.BucketCache(478): Starting BucketCachePersisterThread 2024-11-24T16:45:22,164 INFO [Time-limited test {}] bucket.BucketCachePersister(38): BucketCachePersister started with interval: 100 2024-11-24T16:45:22,165 INFO [Thread-206 {}] bucket.BucketCache(1530): Started retrieving bucket cache from file 2024-11-24T16:45:22,165 INFO [Time-limited test {}] bucket.BucketCache(403): Started bucket cache; ioengine=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/98608d84-f0c0-ab20-75c0-76a28c1904b5/bucket.cache, capacity=400 MB, blockSize=64 KB, writerThreadNum=3, writerQLen=64, persistencePath=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/98608d84-f0c0-ab20-75c0-76a28c1904b5/bucket.persistence, bucketAllocator=org.apache.hadoop.hbase.io.hfile.bucket.BucketAllocator 2024-11-24T16:45:22,165 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-24T16:45:22,165 DEBUG [Time-limited test {}] zookeeper.ZKUtil(111): regionserver:41507-0x1014ade23570003, quorum=127.0.0.1:63542, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-11-24T16:45:22,166 INFO [Thread-206 {}] bucket.BucketCache(1552): Reading new chunked format of persistence. 2024-11-24T16:45:22,167 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:41507-0x1014ade23570003, quorum=127.0.0.1:63542, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-24T16:45:22,167 INFO [Thread-206 {}] bucket.BucketCache(1705): Retrieved 1 of chunks with blockCount = 1. 2024-11-24T16:45:22,167 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=41507 2024-11-24T16:45:22,168 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=41507 2024-11-24T16:45:22,171 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=41507 2024-11-24T16:45:22,171 DEBUG [Thread-206 {}] bucket.BucketCache(1569): Updating region size map after retrieving cached file list 2024-11-24T16:45:22,171 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=41507 2024-11-24T16:45:22,171 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=41507 2024-11-24T16:45:22,172 INFO [Thread-206 {}] bucket.BucketAllocator(343): Cache totalSize=418148352, buckets=199, bucket capacity=2101248=(4*525312)=(FEWEST_ITEMS_IN_BUCKET*(largest configured bucketcache size)) 2024-11-24T16:45:22,172 INFO [Thread-206 {}] bucket.BucketCache(1562): Bucket cache retrieved from file successfully with size: 1 2024-11-24T16:45:22,172 INFO [Thread-206 {}] bucket.BucketCache(413): Persistent bucket cache recovery from /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/98608d84-f0c0-ab20-75c0-76a28c1904b5/bucket.persistence is complete. 2024-11-24T16:45:22,173 INFO [RS:2;e335232f7491:41507 {}] regionserver.HRegionServer(746): ClusterId : 8b6ab050-aca9-4038-b19c-0b3323afc835 2024-11-24T16:45:22,173 DEBUG [RS:2;e335232f7491:41507 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-24T16:45:22,176 DEBUG [RS:2;e335232f7491:41507 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-24T16:45:22,176 DEBUG [RS:2;e335232f7491:41507 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-24T16:45:22,179 DEBUG [RS:2;e335232f7491:41507 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-24T16:45:22,179 DEBUG [RS:2;e335232f7491:41507 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2f347dea, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=e335232f7491/172.17.0.2:0 2024-11-24T16:45:22,189 DEBUG [RSProcedureDispatcher-pool-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-24T16:45:22,191 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-7-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53653, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-24T16:45:22,193 INFO [RS_OPEN_META-regionserver/e335232f7491:0-0 {event_type=M_RS_OPEN_META, pid=9}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-11-24T16:45:22,193 INFO [RS_OPEN_META-regionserver/e335232f7491:0-0 {event_type=M_RS_OPEN_META, pid=9}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-24T16:45:22,194 INFO [RS_OPEN_META-regionserver/e335232f7491:0-0 {event_type=M_RS_OPEN_META, pid=9}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-11-24T16:45:22,195 INFO [RS_OPEN_META-regionserver/e335232f7491:0-0 {event_type=M_RS_OPEN_META, pid=9}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=e335232f7491%2C33045%2C1732466718984.meta, suffix=.meta, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9797cf4c-7cf0-a873-8ba7-f30267b4a2f7/39f4fdd4-f188-1d18-4286-5b0c4dc27cad/WALs/e335232f7491,33045,1732466718984, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9797cf4c-7cf0-a873-8ba7-f30267b4a2f7/39f4fdd4-f188-1d18-4286-5b0c4dc27cad/oldWALs, maxLogs=55 2024-11-24T16:45:22,197 DEBUG [RS:2;e335232f7491:41507 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:2;e335232f7491:41507 2024-11-24T16:45:22,197 INFO [RS:2;e335232f7491:41507 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-24T16:45:22,197 INFO [RS:2;e335232f7491:41507 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-24T16:45:22,197 DEBUG [RS:2;e335232f7491:41507 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-24T16:45:22,198 INFO [RS:2;e335232f7491:41507 {}] regionserver.HRegionServer(2659): reportForDuty to master=e335232f7491,36357,1732466718936 with port=41507, startcode=1732466722156 2024-11-24T16:45:22,198 DEBUG [RS:2;e335232f7491:41507 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-24T16:45:22,200 INFO [HMaster-EventLoopGroup-6-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43069, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=RegionServerStatusService 2024-11-24T16:45:22,201 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=36357 {}] master.ServerManager(363): Checking decommissioned status of RegionServer e335232f7491,41507,1732466722156 2024-11-24T16:45:22,201 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=36357 {}] master.ServerManager(517): Registering regionserver=e335232f7491,41507,1732466722156 2024-11-24T16:45:22,202 DEBUG [RS:2;e335232f7491:41507 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9797cf4c-7cf0-a873-8ba7-f30267b4a2f7/39f4fdd4-f188-1d18-4286-5b0c4dc27cad 2024-11-24T16:45:22,202 DEBUG [RS:2;e335232f7491:41507 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=file:/ 2024-11-24T16:45:22,202 DEBUG [RS:2;e335232f7491:41507 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-24T16:45:22,205 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36357-0x1014ade23570000, quorum=127.0.0.1:63542, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-24T16:45:22,207 DEBUG [RS:2;e335232f7491:41507 {}] zookeeper.ZKUtil(111): regionserver:41507-0x1014ade23570003, quorum=127.0.0.1:63542, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/e335232f7491,41507,1732466722156 2024-11-24T16:45:22,207 WARN [RS:2;e335232f7491:41507 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-24T16:45:22,207 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [e335232f7491,41507,1732466722156] 2024-11-24T16:45:22,207 INFO [RS:2;e335232f7491:41507 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-24T16:45:22,207 DEBUG [RS:2;e335232f7491:41507 {}] regionserver.HRegionServer(1793): logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9797cf4c-7cf0-a873-8ba7-f30267b4a2f7/39f4fdd4-f188-1d18-4286-5b0c4dc27cad/WALs/e335232f7491,41507,1732466722156 2024-11-24T16:45:22,213 INFO [RS:2;e335232f7491:41507 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-24T16:45:22,216 INFO [RS:2;e335232f7491:41507 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-24T16:45:22,217 INFO [RS:2;e335232f7491:41507 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-24T16:45:22,217 INFO [RS:2;e335232f7491:41507 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-24T16:45:22,217 INFO [RS:2;e335232f7491:41507 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-24T16:45:22,218 INFO [RS:2;e335232f7491:41507 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-24T16:45:22,218 INFO [RS:2;e335232f7491:41507 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-24T16:45:22,218 DEBUG [RS:2;e335232f7491:41507 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/e335232f7491:0, corePoolSize=1, maxPoolSize=1 2024-11-24T16:45:22,218 DEBUG [RS:2;e335232f7491:41507 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/e335232f7491:0, corePoolSize=1, maxPoolSize=1 2024-11-24T16:45:22,218 DEBUG [RS:2;e335232f7491:41507 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/e335232f7491:0, corePoolSize=1, maxPoolSize=1 2024-11-24T16:45:22,218 DEBUG [RS:2;e335232f7491:41507 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/e335232f7491:0, corePoolSize=1, maxPoolSize=1 2024-11-24T16:45:22,218 DEBUG [RS:2;e335232f7491:41507 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/e335232f7491:0, corePoolSize=1, maxPoolSize=1 2024-11-24T16:45:22,218 DEBUG [RS:2;e335232f7491:41507 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/e335232f7491:0, corePoolSize=2, maxPoolSize=2 2024-11-24T16:45:22,218 DEBUG [RS:2;e335232f7491:41507 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/e335232f7491:0, corePoolSize=1, maxPoolSize=1 2024-11-24T16:45:22,218 DEBUG [RS:2;e335232f7491:41507 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/e335232f7491:0, corePoolSize=1, maxPoolSize=1 2024-11-24T16:45:22,218 DEBUG [RS:2;e335232f7491:41507 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/e335232f7491:0, corePoolSize=1, maxPoolSize=1 2024-11-24T16:45:22,218 DEBUG [RS:2;e335232f7491:41507 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/e335232f7491:0, corePoolSize=1, maxPoolSize=1 2024-11-24T16:45:22,218 DEBUG [RS:2;e335232f7491:41507 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/e335232f7491:0, corePoolSize=1, maxPoolSize=1 2024-11-24T16:45:22,219 DEBUG [RS:2;e335232f7491:41507 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/e335232f7491:0, corePoolSize=1, maxPoolSize=1 2024-11-24T16:45:22,219 DEBUG [RS:2;e335232f7491:41507 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/e335232f7491:0, corePoolSize=3, maxPoolSize=3 2024-11-24T16:45:22,219 DEBUG [RS:2;e335232f7491:41507 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0, corePoolSize=3, maxPoolSize=3 2024-11-24T16:45:22,223 INFO [RS:2;e335232f7491:41507 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-24T16:45:22,223 INFO [RS:2;e335232f7491:41507 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-24T16:45:22,223 INFO [RS:2;e335232f7491:41507 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-24T16:45:22,224 INFO [RS:2;e335232f7491:41507 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-24T16:45:22,224 INFO [RS:2;e335232f7491:41507 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-24T16:45:22,224 INFO [RS:2;e335232f7491:41507 {}] hbase.ChoreService(168): Chore ScheduledChore name=e335232f7491,41507,1732466722156-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-24T16:45:22,227 INFO [RS_OPEN_META-regionserver/e335232f7491:0-0 {event_type=M_RS_OPEN_META, pid=9}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9797cf4c-7cf0-a873-8ba7-f30267b4a2f7/39f4fdd4-f188-1d18-4286-5b0c4dc27cad/WALs/e335232f7491,33045,1732466718984/e335232f7491%2C33045%2C1732466718984.meta.1732466722196.meta 2024-11-24T16:45:22,228 DEBUG [RS_OPEN_META-regionserver/e335232f7491:0-0 {event_type=M_RS_OPEN_META, pid=9}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-11-24T16:45:22,228 DEBUG [RS_OPEN_META-regionserver/e335232f7491:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-11-24T16:45:22,228 DEBUG [RS_OPEN_META-regionserver/e335232f7491:0-0 {event_type=M_RS_OPEN_META, pid=9}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-11-24T16:45:22,229 DEBUG [RS_OPEN_META-regionserver/e335232f7491:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-11-24T16:45:22,229 INFO [RS_OPEN_META-regionserver/e335232f7491:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-11-24T16:45:22,229 DEBUG [RS_OPEN_META-regionserver/e335232f7491:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-11-24T16:45:22,229 DEBUG [RS_OPEN_META-regionserver/e335232f7491:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T16:45:22,229 DEBUG [RS_OPEN_META-regionserver/e335232f7491:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-11-24T16:45:22,229 DEBUG [RS_OPEN_META-regionserver/e335232f7491:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-11-24T16:45:22,235 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family info of region 1588230740 2024-11-24T16:45:22,236 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-24T16:45:22,236 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:45:22,238 DEBUG [StoreFileOpener-1588230740-info-1 {}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9797cf4c-7cf0-a873-8ba7-f30267b4a2f7/39f4fdd4-f188-1d18-4286-5b0c4dc27cad/data/hbase/meta/1588230740/info/971b3bd28c36406c9e7e88e8e2172769, delay=941 ms 2024-11-24T16:45:22,239 DEBUG [StoreOpener-1588230740-1 {}] regionserver.StoreEngine(278): loaded file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9797cf4c-7cf0-a873-8ba7-f30267b4a2f7/39f4fdd4-f188-1d18-4286-5b0c4dc27cad/data/hbase/meta/1588230740/info/971b3bd28c36406c9e7e88e8e2172769 2024-11-24T16:45:22,239 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-24T16:45:22,239 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family ns of region 1588230740 2024-11-24T16:45:22,240 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-24T16:45:22,240 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:45:22,242 DEBUG [StoreFileOpener-1588230740-ns-1 {}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9797cf4c-7cf0-a873-8ba7-f30267b4a2f7/39f4fdd4-f188-1d18-4286-5b0c4dc27cad/data/hbase/meta/1588230740/ns/c1c6018341f54195884d54ad8cd64f1d, delay=937 ms 2024-11-24T16:45:22,244 INFO [RS:2;e335232f7491:41507 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-24T16:45:22,244 DEBUG [StoreOpener-1588230740-1 {}] regionserver.StoreEngine(278): loaded file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9797cf4c-7cf0-a873-8ba7-f30267b4a2f7/39f4fdd4-f188-1d18-4286-5b0c4dc27cad/data/hbase/meta/1588230740/ns/c1c6018341f54195884d54ad8cd64f1d 2024-11-24T16:45:22,244 INFO [RS:2;e335232f7491:41507 {}] hbase.ChoreService(168): Chore ScheduledChore name=e335232f7491,41507,1732466722156-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-24T16:45:22,244 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-24T16:45:22,244 INFO [RS:2;e335232f7491:41507 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-24T16:45:22,244 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family rep_barrier of region 1588230740 2024-11-24T16:45:22,244 INFO [RS:2;e335232f7491:41507 {}] regionserver.Replication(171): e335232f7491,41507,1732466722156 started 2024-11-24T16:45:22,245 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-24T16:45:22,245 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:45:22,245 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-24T16:45:22,246 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family table of region 1588230740 2024-11-24T16:45:22,246 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-24T16:45:22,246 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:45:22,249 DEBUG [StoreFileOpener-1588230740-table-1 {}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9797cf4c-7cf0-a873-8ba7-f30267b4a2f7/39f4fdd4-f188-1d18-4286-5b0c4dc27cad/data/hbase/meta/1588230740/table/6ee5d4979983472db67dd250d09f8b2a, delay=956 ms 2024-11-24T16:45:22,250 DEBUG [StoreOpener-1588230740-1 {}] regionserver.StoreEngine(278): loaded file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9797cf4c-7cf0-a873-8ba7-f30267b4a2f7/39f4fdd4-f188-1d18-4286-5b0c4dc27cad/data/hbase/meta/1588230740/table/6ee5d4979983472db67dd250d09f8b2a 2024-11-24T16:45:22,250 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-24T16:45:22,251 DEBUG [RS_OPEN_META-regionserver/e335232f7491:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-24T16:45:22,252 DEBUG [RS_OPEN_META-regionserver/e335232f7491:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9797cf4c-7cf0-a873-8ba7-f30267b4a2f7/39f4fdd4-f188-1d18-4286-5b0c4dc27cad/data/hbase/meta/1588230740 2024-11-24T16:45:22,252 DEBUG [RS_OPEN_META-regionserver/e335232f7491:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9797cf4c-7cf0-a873-8ba7-f30267b4a2f7/39f4fdd4-f188-1d18-4286-5b0c4dc27cad/data/hbase/meta/1588230740 2024-11-24T16:45:22,253 DEBUG [RS_OPEN_META-regionserver/e335232f7491:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-24T16:45:22,253 DEBUG [RS_OPEN_META-regionserver/e335232f7491:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-24T16:45:22,254 DEBUG [RS_OPEN_META-regionserver/e335232f7491:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-24T16:45:22,255 DEBUG [RS_OPEN_META-regionserver/e335232f7491:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-24T16:45:22,256 INFO [RS_OPEN_META-regionserver/e335232f7491:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=15; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=71609858, jitterRate=0.06707003712654114}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-24T16:45:22,256 DEBUG [RS_OPEN_META-regionserver/e335232f7491:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-11-24T16:45:22,257 DEBUG [RS_OPEN_META-regionserver/e335232f7491:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1732466722229Writing region info on filesystem at 1732466722229Initializing all the Stores at 1732466722230 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732466722230Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732466722235 (+5 ms)Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732466722235Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732466722235Cleaning up temporary data from old regions at 1732466722253 (+18 ms)Running coprocessor post-open hooks at 1732466722256 (+3 ms)Region opened successfully at 1732466722257 (+1 ms) 2024-11-24T16:45:22,258 INFO [RS_OPEN_META-regionserver/e335232f7491:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=9, masterSystemTime=1732466722189 2024-11-24T16:45:22,261 DEBUG [RS_OPEN_META-regionserver/e335232f7491:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-11-24T16:45:22,261 INFO [RS_OPEN_META-regionserver/e335232f7491:0-0 {event_type=M_RS_OPEN_META, pid=9}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-11-24T16:45:22,261 INFO [RS:2;e335232f7491:41507 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-24T16:45:22,261 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=8 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=15, regionLocation=e335232f7491,33045,1732466718984 2024-11-24T16:45:22,261 INFO [RS:2;e335232f7491:41507 {}] regionserver.HRegionServer(1482): Serving as e335232f7491,41507,1732466722156, RpcServer on e335232f7491/172.17.0.2:41507, sessionid=0x1014ade23570003 2024-11-24T16:45:22,262 DEBUG [RS:2;e335232f7491:41507 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-24T16:45:22,262 DEBUG [RS:2;e335232f7491:41507 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager e335232f7491,41507,1732466722156 2024-11-24T16:45:22,262 DEBUG [RS:2;e335232f7491:41507 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'e335232f7491,41507,1732466722156' 2024-11-24T16:45:22,262 DEBUG [RS:2;e335232f7491:41507 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-24T16:45:22,262 INFO [PEWorker-1 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as e335232f7491,33045,1732466718984, state=OPEN 2024-11-24T16:45:22,262 DEBUG [RS:2;e335232f7491:41507 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-24T16:45:22,264 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41507-0x1014ade23570003, quorum=127.0.0.1:63542, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-24T16:45:22,264 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33045-0x1014ade23570001, quorum=127.0.0.1:63542, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-24T16:45:22,264 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36357-0x1014ade23570000, quorum=127.0.0.1:63542, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-24T16:45:22,264 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-24T16:45:22,264 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-24T16:45:22,264 DEBUG [PEWorker-1 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=9, ppid=8, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=e335232f7491,33045,1732466718984 2024-11-24T16:45:22,264 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-24T16:45:22,264 DEBUG [RS:2;e335232f7491:41507 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-24T16:45:22,264 DEBUG [RS:2;e335232f7491:41507 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-24T16:45:22,264 DEBUG [RS:2;e335232f7491:41507 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager e335232f7491,41507,1732466722156 2024-11-24T16:45:22,264 DEBUG [RS:2;e335232f7491:41507 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'e335232f7491,41507,1732466722156' 2024-11-24T16:45:22,264 DEBUG [RS:2;e335232f7491:41507 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-24T16:45:22,265 DEBUG [RS:2;e335232f7491:41507 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-24T16:45:22,265 DEBUG [RS:2;e335232f7491:41507 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-24T16:45:22,266 INFO [RS:2;e335232f7491:41507 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-24T16:45:22,266 INFO [RS:2;e335232f7491:41507 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-24T16:45:22,266 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=9, resume processing ppid=8 2024-11-24T16:45:22,266 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=9, ppid=8, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=e335232f7491,33045,1732466718984 in 227 msec 2024-11-24T16:45:22,268 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=8, resume processing ppid=7 2024-11-24T16:45:22,268 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=8, ppid=7, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 385 msec 2024-11-24T16:45:22,269 INFO [PEWorker-3 {}] procedure.ServerCrashProcedure(207): e335232f7491,42963,1732466719025 had 2 regions 2024-11-24T16:45:22,269 INFO [PEWorker-3 {}] procedure.ServerCrashProcedure(339): Splitting WALs pid=7, state=RUNNABLE:SERVER_CRASH_SPLIT_LOGS, hasLock=true; ServerCrashProcedure e335232f7491,42963,1732466719025, splitWal=true, meta=true, isMeta: false 2024-11-24T16:45:22,270 INFO [PEWorker-3 {}] master.SplitLogManager(171): file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9797cf4c-7cf0-a873-8ba7-f30267b4a2f7/39f4fdd4-f188-1d18-4286-5b0c4dc27cad/WALs/e335232f7491,42963,1732466719025-splitting dir is empty, no logs to split. 2024-11-24T16:45:22,270 INFO [PEWorker-3 {}] master.SplitWALManager(105): e335232f7491,42963,1732466719025 WAL count=0, meta=false 2024-11-24T16:45:22,271 INFO [PEWorker-3 {}] master.SplitLogManager(171): file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9797cf4c-7cf0-a873-8ba7-f30267b4a2f7/39f4fdd4-f188-1d18-4286-5b0c4dc27cad/WALs/e335232f7491,42963,1732466719025-splitting dir is empty, no logs to split. 2024-11-24T16:45:22,271 INFO [PEWorker-3 {}] master.SplitWALManager(105): e335232f7491,42963,1732466719025 WAL count=0, meta=false 2024-11-24T16:45:22,271 DEBUG [PEWorker-3 {}] procedure.ServerCrashProcedure(329): Check if e335232f7491,42963,1732466719025 WAL splitting is done? wals=0, meta=false 2024-11-24T16:45:22,272 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=10, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1732466719962, region=edbfb0487dd5f5fb1ea5f080d86356e0, ASSIGN}] 2024-11-24T16:45:22,273 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=10, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1732466719962, region=edbfb0487dd5f5fb1ea5f080d86356e0, ASSIGN 2024-11-24T16:45:22,274 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=10, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1732466719962, region=edbfb0487dd5f5fb1ea5f080d86356e0, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-11-24T16:45:22,366 INFO [RS:2;e335232f7491:41507 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-24T16:45:22,367 INFO [RS:2;e335232f7491:41507 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=e335232f7491%2C41507%2C1732466722156, suffix=, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9797cf4c-7cf0-a873-8ba7-f30267b4a2f7/39f4fdd4-f188-1d18-4286-5b0c4dc27cad/WALs/e335232f7491,41507,1732466722156, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9797cf4c-7cf0-a873-8ba7-f30267b4a2f7/39f4fdd4-f188-1d18-4286-5b0c4dc27cad/oldWALs, maxLogs=55 2024-11-24T16:45:22,386 INFO [RS:2;e335232f7491:41507 {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9797cf4c-7cf0-a873-8ba7-f30267b4a2f7/39f4fdd4-f188-1d18-4286-5b0c4dc27cad/WALs/e335232f7491,41507,1732466722156/e335232f7491%2C41507%2C1732466722156.1732466722368 2024-11-24T16:45:22,387 DEBUG [RS:2;e335232f7491:41507 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-11-24T16:45:22,424 DEBUG [e335232f7491:36357 {}] balancer.BalancerClusterState(204): Hosts are {e335232f7491=0} racks are {/default-rack=0} 2024-11-24T16:45:22,425 DEBUG [e335232f7491:36357 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-11-24T16:45:22,425 DEBUG [e335232f7491:36357 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-11-24T16:45:22,425 DEBUG [e335232f7491:36357 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-11-24T16:45:22,425 DEBUG [e335232f7491:36357 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-11-24T16:45:22,425 INFO [e335232f7491:36357 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-11-24T16:45:22,425 INFO [e335232f7491:36357 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-11-24T16:45:22,425 DEBUG [e335232f7491:36357 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-11-24T16:45:22,425 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=10 updating hbase:meta row=edbfb0487dd5f5fb1ea5f080d86356e0, regionState=OPENING, regionLocation=e335232f7491,41507,1732466722156 2024-11-24T16:45:22,426 WARN [MiniHBaseClusterRegionServer-EventLoopGroup-8-3 {}] ipc.NettyRpcConnection$2(409): Exception encountered while connecting to the server e335232f7491:42963 org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: e335232f7491/172.17.0.2:42963 Caused by: java.net.ConnectException: finishConnect(..) failed: Connection refused at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.newConnectException0(Errors.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.handleConnectErrno(Errors.java:131) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Socket.finishConnect(Socket.java:359) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.doFinishConnect(AbstractEpollChannel.java:715) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:692) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T16:45:22,428 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-8-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=e335232f7491,42963,1732466719025, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=e335232f7491,42963,1732466719025, seqNum=-1, error=java.net.ConnectException: Call to address=e335232f7491:42963 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: e335232f7491/172.17.0.2:42963 2024-11-24T16:45:22,429 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-8-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:meta,,1.1588230740, hostname=e335232f7491,42963,1732466719025, seqNum=-1 is java.net.ConnectException: finishConnect(..) failed: Connection refused 2024-11-24T16:45:22,429 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-8-3 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:meta,,1.1588230740, hostname=e335232f7491,42963,1732466719025, seqNum=-1 from cache 2024-11-24T16:45:22,430 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-8-3 {}] ipc.FailedServers(52): Added failed server with address e335232f7491:42963 to list caused by org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: e335232f7491/172.17.0.2:42963 2024-11-24T16:45:22,536 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-24T16:45:22,536 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=e335232f7491,33045,1732466718984, seqNum=-1] 2024-11-24T16:45:22,537 DEBUG [Async-Client-Retry-Timer-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-24T16:45:22,539 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-7-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45841, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-24T16:45:22,541 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-8-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=10, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1732466719962, region=edbfb0487dd5f5fb1ea5f080d86356e0, ASSIGN because future has completed 2024-11-24T16:45:22,541 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=11, ppid=10, state=RUNNABLE, hasLock=false; OpenRegionProcedure edbfb0487dd5f5fb1ea5f080d86356e0, server=e335232f7491,41507,1732466722156}] 2024-11-24T16:45:22,693 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-24T16:45:22,695 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-9-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:55819, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-24T16:45:22,698 INFO [RS_OPEN_REGION-regionserver/e335232f7491:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] handler.AssignRegionHandler(132): Open testBlockEvictionOnGracefulStop1732466719962,,1732466719966.edbfb0487dd5f5fb1ea5f080d86356e0. 2024-11-24T16:45:22,698 DEBUG [RS_OPEN_REGION-regionserver/e335232f7491:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(7752): Opening region: {ENCODED => edbfb0487dd5f5fb1ea5f080d86356e0, NAME => 'testBlockEvictionOnGracefulStop1732466719962,,1732466719966.edbfb0487dd5f5fb1ea5f080d86356e0.', STARTKEY => '', ENDKEY => ''} 2024-11-24T16:45:22,699 DEBUG [RS_OPEN_REGION-regionserver/e335232f7491:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testBlockEvictionOnGracefulStop1732466719962 edbfb0487dd5f5fb1ea5f080d86356e0 2024-11-24T16:45:22,699 DEBUG [RS_OPEN_REGION-regionserver/e335232f7491:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(898): Instantiated testBlockEvictionOnGracefulStop1732466719962,,1732466719966.edbfb0487dd5f5fb1ea5f080d86356e0.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T16:45:22,699 DEBUG [RS_OPEN_REGION-regionserver/e335232f7491:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(7794): checking encryption for edbfb0487dd5f5fb1ea5f080d86356e0 2024-11-24T16:45:22,699 DEBUG [RS_OPEN_REGION-regionserver/e335232f7491:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(7797): checking classloading for edbfb0487dd5f5fb1ea5f080d86356e0 2024-11-24T16:45:22,700 INFO [StoreOpener-edbfb0487dd5f5fb1ea5f080d86356e0-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family family of region edbfb0487dd5f5fb1ea5f080d86356e0 2024-11-24T16:45:22,700 INFO [StoreOpener-edbfb0487dd5f5fb1ea5f080d86356e0-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region edbfb0487dd5f5fb1ea5f080d86356e0 columnFamilyName family 2024-11-24T16:45:22,700 DEBUG [StoreOpener-edbfb0487dd5f5fb1ea5f080d86356e0-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:45:22,702 DEBUG [StoreFileOpener-edbfb0487dd5f5fb1ea5f080d86356e0-family-1 {}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9797cf4c-7cf0-a873-8ba7-f30267b4a2f7/39f4fdd4-f188-1d18-4286-5b0c4dc27cad/data/default/testBlockEvictionOnGracefulStop1732466719962/edbfb0487dd5f5fb1ea5f080d86356e0/family/1ebd1eb39df242bbae9f0d38b310e39f, delay=947 ms 2024-11-24T16:45:22,704 DEBUG [StoreOpener-edbfb0487dd5f5fb1ea5f080d86356e0-1 {}] regionserver.StoreEngine(278): loaded file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9797cf4c-7cf0-a873-8ba7-f30267b4a2f7/39f4fdd4-f188-1d18-4286-5b0c4dc27cad/data/default/testBlockEvictionOnGracefulStop1732466719962/edbfb0487dd5f5fb1ea5f080d86356e0/family/1ebd1eb39df242bbae9f0d38b310e39f 2024-11-24T16:45:22,704 INFO [StoreOpener-edbfb0487dd5f5fb1ea5f080d86356e0-1 {}] regionserver.HStore(327): Store=edbfb0487dd5f5fb1ea5f080d86356e0/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T16:45:22,704 DEBUG [RS_OPEN_REGION-regionserver/e335232f7491:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(1038): replaying wal for edbfb0487dd5f5fb1ea5f080d86356e0 2024-11-24T16:45:22,705 DEBUG [RS_OPEN_REGION-regionserver/e335232f7491:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9797cf4c-7cf0-a873-8ba7-f30267b4a2f7/39f4fdd4-f188-1d18-4286-5b0c4dc27cad/data/default/testBlockEvictionOnGracefulStop1732466719962/edbfb0487dd5f5fb1ea5f080d86356e0 2024-11-24T16:45:22,705 DEBUG [RS_OPEN_REGION-regionserver/e335232f7491:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9797cf4c-7cf0-a873-8ba7-f30267b4a2f7/39f4fdd4-f188-1d18-4286-5b0c4dc27cad/data/default/testBlockEvictionOnGracefulStop1732466719962/edbfb0487dd5f5fb1ea5f080d86356e0 2024-11-24T16:45:22,705 DEBUG [RS_OPEN_REGION-regionserver/e335232f7491:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(1048): stopping wal replay for edbfb0487dd5f5fb1ea5f080d86356e0 2024-11-24T16:45:22,705 DEBUG [RS_OPEN_REGION-regionserver/e335232f7491:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(1060): Cleaning up temporary data for edbfb0487dd5f5fb1ea5f080d86356e0 2024-11-24T16:45:22,706 DEBUG [RS_OPEN_REGION-regionserver/e335232f7491:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(1093): writing seq id for edbfb0487dd5f5fb1ea5f080d86356e0 2024-11-24T16:45:22,707 INFO [RS_OPEN_REGION-regionserver/e335232f7491:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(1114): Opened edbfb0487dd5f5fb1ea5f080d86356e0; next sequenceid=10; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=68373263, jitterRate=0.018841013312339783}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-24T16:45:22,707 DEBUG [RS_OPEN_REGION-regionserver/e335232f7491:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(1122): Running coprocessor post-open hooks for edbfb0487dd5f5fb1ea5f080d86356e0 2024-11-24T16:45:22,708 DEBUG [RS_OPEN_REGION-regionserver/e335232f7491:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(1006): Region open journal for edbfb0487dd5f5fb1ea5f080d86356e0: Running coprocessor pre-open hook at 1732466722699Writing region info on filesystem at 1732466722699Initializing all the Stores at 1732466722699Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732466722699Cleaning up temporary data from old regions at 1732466722705 (+6 ms)Running coprocessor post-open hooks at 1732466722707 (+2 ms)Region opened successfully at 1732466722708 (+1 ms) 2024-11-24T16:45:22,709 INFO [RS_OPEN_REGION-regionserver/e335232f7491:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegionServer(2236): Post open deploy tasks for testBlockEvictionOnGracefulStop1732466719962,,1732466719966.edbfb0487dd5f5fb1ea5f080d86356e0., pid=11, masterSystemTime=1732466722693 2024-11-24T16:45:22,711 DEBUG [RS_OPEN_REGION-regionserver/e335232f7491:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegionServer(2266): Finished post open deploy task for testBlockEvictionOnGracefulStop1732466719962,,1732466719966.edbfb0487dd5f5fb1ea5f080d86356e0. 2024-11-24T16:45:22,711 INFO [RS_OPEN_REGION-regionserver/e335232f7491:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] handler.AssignRegionHandler(153): Opened testBlockEvictionOnGracefulStop1732466719962,,1732466719966.edbfb0487dd5f5fb1ea5f080d86356e0. 2024-11-24T16:45:22,711 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=10 updating hbase:meta row=edbfb0487dd5f5fb1ea5f080d86356e0, regionState=OPEN, openSeqNum=10, regionLocation=e335232f7491,41507,1732466722156 2024-11-24T16:45:22,714 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-8-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=11, ppid=10, state=RUNNABLE, hasLock=false; OpenRegionProcedure edbfb0487dd5f5fb1ea5f080d86356e0, server=e335232f7491,41507,1732466722156 because future has completed 2024-11-24T16:45:22,717 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=11, resume processing ppid=10 2024-11-24T16:45:22,717 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=11, ppid=10, state=SUCCESS, hasLock=false; OpenRegionProcedure edbfb0487dd5f5fb1ea5f080d86356e0, server=e335232f7491,41507,1732466722156 in 174 msec 2024-11-24T16:45:22,719 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=10, resume processing ppid=7 2024-11-24T16:45:22,719 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=10, ppid=7, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1732466719962, region=edbfb0487dd5f5fb1ea5f080d86356e0, ASSIGN in 445 msec 2024-11-24T16:45:22,719 INFO [PEWorker-1 {}] procedure.ServerCrashProcedure(291): removed crashed server e335232f7491,42963,1732466719025 after splitting done 2024-11-24T16:45:22,721 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=7, state=SUCCESS, hasLock=false; ServerCrashProcedure e335232f7491,42963,1732466719025, splitWal=true, meta=true in 849 msec 2024-11-24T16:45:22,762 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster 2024-11-24T16:45:22,762 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-24T16:45:22,762 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79) at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611) at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.io.hfile.TestBlockEvictionOnRegionMovement.tearDown(TestBlockEvictionOnRegionMovement.java:174) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-24T16:45:22,762 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T16:45:22,762 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T16:45:22,762 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-24T16:45:22,762 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2024-11-24T16:45:22,762 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=1634292174, stopped=false 2024-11-24T16:45:22,762 INFO [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=e335232f7491,36357,1732466718936 2024-11-24T16:45:22,765 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41507-0x1014ade23570003, quorum=127.0.0.1:63542, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-24T16:45:22,765 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33045-0x1014ade23570001, quorum=127.0.0.1:63542, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-24T16:45:22,765 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36357-0x1014ade23570000, quorum=127.0.0.1:63542, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-24T16:45:22,765 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41507-0x1014ade23570003, quorum=127.0.0.1:63542, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T16:45:22,765 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-24T16:45:22,765 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36357-0x1014ade23570000, quorum=127.0.0.1:63542, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T16:45:22,765 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33045-0x1014ade23570001, quorum=127.0.0.1:63542, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T16:45:22,766 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-24T16:45:22,766 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277) at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265) at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416) at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.io.hfile.TestBlockEvictionOnRegionMovement.tearDown(TestBlockEvictionOnRegionMovement.java:174) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-24T16:45:22,766 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T16:45:22,766 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:41507-0x1014ade23570003, quorum=127.0.0.1:63542, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-24T16:45:22,766 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:36357-0x1014ade23570000, quorum=127.0.0.1:63542, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-24T16:45:22,766 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:33045-0x1014ade23570001, quorum=127.0.0.1:63542, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-24T16:45:22,766 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server 'e335232f7491,33045,1732466718984' ***** 2024-11-24T16:45:22,766 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-11-24T16:45:22,766 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server 'e335232f7491,41507,1732466722156' ***** 2024-11-24T16:45:22,766 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-11-24T16:45:22,766 INFO [RS:0;e335232f7491:33045 {}] bucket.BucketCache(1738): Disabling cache 2024-11-24T16:45:22,766 INFO [RS:2;e335232f7491:41507 {}] bucket.BucketCache(1738): Disabling cache 2024-11-24T16:45:22,767 INFO [RS:0;e335232f7491:33045 {}] bucket.BucketCache(1762): Shutdown bucket cache: IO persistent=true; path to write=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/98608d84-f0c0-ab20-75c0-76a28c1904b5/bucket.persistence 2024-11-24T16:45:22,767 INFO [Time-limited test-BucketCacheWriter-1 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-1 exiting, cacheEnabled=false 2024-11-24T16:45:22,767 INFO [Time-limited test-BucketCacheWriter-0 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-0 exiting, cacheEnabled=false 2024-11-24T16:45:22,767 INFO [Time-limited test-BucketCacheWriter-2 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-2 exiting, cacheEnabled=false 2024-11-24T16:45:22,767 INFO [RS:0;e335232f7491:33045 {}] bucket.BucketCache(1768): Shutting down cache persister thread. 2024-11-24T16:45:22,767 INFO [Time-limited test-BucketCacheWriter-1 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-1 exiting, cacheEnabled=false 2024-11-24T16:45:22,767 INFO [Time-limited test-BucketCacheWriter-0 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-0 exiting, cacheEnabled=false 2024-11-24T16:45:22,767 INFO [RS:2;e335232f7491:41507 {}] bucket.BucketCache(1762): Shutdown bucket cache: IO persistent=true; path to write=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/98608d84-f0c0-ab20-75c0-76a28c1904b5/bucket.persistence 2024-11-24T16:45:22,767 INFO [Time-limited test-BucketCacheWriter-2 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-2 exiting, cacheEnabled=false 2024-11-24T16:45:22,767 WARN [bucket-cache-persister {}] bucket.BucketCachePersister(64): Interrupting BucketCachePersister thread. java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCachePersister.run(BucketCachePersister.java:45) ~[classes/:?] 2024-11-24T16:45:22,767 INFO [RS:2;e335232f7491:41507 {}] bucket.BucketCache(1768): Shutting down cache persister thread. 2024-11-24T16:45:22,768 WARN [bucket-cache-persister {}] bucket.BucketCachePersister(64): Interrupting BucketCachePersister thread. java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCachePersister.run(BucketCachePersister.java:45) ~[classes/:?] 2024-11-24T16:45:22,777 DEBUG [RS:0;e335232f7491:33045 {}] bucket.BucketCache(1487): Thread RS:0;e335232f7491:33045 started persisting bucket cache to file 2024-11-24T16:45:22,777 DEBUG [RS:0;e335232f7491:33045 {}] bucket.BucketCache(1494): Persist in new chunked persistence format. 2024-11-24T16:45:22,777 DEBUG [RS:0;e335232f7491:33045 {}] bucket.BucketCache(1673): persistToFile: before persisting backing map size: 0, fullycachedFiles size: 0, chunkSize: 10000 2024-11-24T16:45:22,778 DEBUG [RS:2;e335232f7491:41507 {}] bucket.BucketCache(1487): Thread RS:2;e335232f7491:41507 started persisting bucket cache to file 2024-11-24T16:45:22,778 DEBUG [RS:2;e335232f7491:41507 {}] bucket.BucketCache(1494): Persist in new chunked persistence format. 2024-11-24T16:45:22,778 DEBUG [RS:2;e335232f7491:41507 {}] bucket.BucketCache(1673): persistToFile: before persisting backing map size: 1, fullycachedFiles size: 0, chunkSize: 10000 2024-11-24T16:45:22,781 DEBUG [RS:0;e335232f7491:33045 {}] bucket.BucketCache(1680): persistToFile: after persisting backing map size: 0, fullycachedFiles size: 0 2024-11-24T16:45:22,781 DEBUG [RS:0;e335232f7491:33045 {}] bucket.BucketCache(1498): PersistToFile: after persisting backing map size: 0, fullycachedFiles size: 0, file name: bucket.persistence1732466722777 2024-11-24T16:45:22,781 DEBUG [RS:0;e335232f7491:33045 {}] bucket.BucketCache(1509): Thread RS:0;e335232f7491:33045 finished persisting bucket cache to file, renaming 2024-11-24T16:45:22,781 INFO [RS:0;e335232f7491:33045 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-24T16:45:22,781 INFO [RS:0;e335232f7491:33045 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-24T16:45:22,781 ERROR [RS:2;e335232f7491:41507 {}] bucket.PersistentIOEngine(78): Calculating checksum failed, because of java.io.IOException: Stream closed at java.io.BufferedInputStream.getBufIfOpen(BufferedInputStream.java:168) ~[?:?] at java.io.BufferedInputStream.read(BufferedInputStream.java:334) ~[?:?] at sun.nio.cs.StreamDecoder.readBytes(StreamDecoder.java:281) ~[?:?] at sun.nio.cs.StreamDecoder.implRead(StreamDecoder.java:324) ~[?:?] at sun.nio.cs.StreamDecoder.read(StreamDecoder.java:189) ~[?:?] at java.io.InputStreamReader.read(InputStreamReader.java:177) ~[?:?] at java.io.BufferedReader.fill(BufferedReader.java:162) ~[?:?] at java.io.BufferedReader.readLine(BufferedReader.java:329) ~[?:?] at java.io.BufferedReader.readLine(BufferedReader.java:396) ~[?:?] at org.apache.hadoop.util.Shell.runCommand(Shell.java:1056) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.util.Shell.run(Shell.java:959) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.util.Shell$ShellCommandExecutor.execute(Shell.java:1282) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.io.hfile.bucket.PersistentIOEngine.getFileSize(PersistentIOEngine.java:94) ~[classes/:?] at org.apache.hadoop.hbase.io.hfile.bucket.PersistentIOEngine.calculateChecksum(PersistentIOEngine.java:71) ~[classes/:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketProtoUtils.toPB(BucketProtoUtils.java:61) ~[classes/:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketProtoUtils.serializeAsPB(BucketProtoUtils.java:75) ~[classes/:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCache.persistChunkedBackingMap(BucketCache.java:1678) ~[classes/:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCache.persistToFile(BucketCache.java:1496) ~[classes/:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCache.shutdown(BucketCache.java:1774) ~[classes/:?] at org.apache.hadoop.hbase.io.hfile.CombinedBlockCache.shutdown(CombinedBlockCache.java:169) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:927) ~[classes/:?] at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) ~[test-classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) ~[test-classes/:4.0.0-alpha-1-SNAPSHOT] at java.security.AccessController.doPrivileged(AccessController.java:399) ~[?:?] at javax.security.auth.Subject.doAs(Subject.java:376) ~[?:?] at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) ~[test-classes/:4.0.0-alpha-1-SNAPSHOT] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T16:45:22,781 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-24T16:45:22,781 INFO [RS:0;e335232f7491:33045 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-24T16:45:22,781 INFO [RS:0;e335232f7491:33045 {}] regionserver.HRegionServer(959): stopping server e335232f7491,33045,1732466718984 2024-11-24T16:45:22,781 INFO [RS:0;e335232f7491:33045 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-24T16:45:22,781 INFO [RS:0;e335232f7491:33045 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;e335232f7491:33045. 2024-11-24T16:45:22,781 DEBUG [RS:2;e335232f7491:41507 {}] bucket.BucketCache(1680): persistToFile: after persisting backing map size: 1, fullycachedFiles size: 0 2024-11-24T16:45:22,781 DEBUG [RS:0;e335232f7491:33045 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-24T16:45:22,782 DEBUG [RS:2;e335232f7491:41507 {}] bucket.BucketCache(1498): PersistToFile: after persisting backing map size: 1, fullycachedFiles size: 0, file name: bucket.persistence1732466722778 2024-11-24T16:45:22,782 DEBUG [RS:0;e335232f7491:33045 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T16:45:22,782 DEBUG [RS:2;e335232f7491:41507 {}] bucket.BucketCache(1509): Thread RS:2;e335232f7491:41507 finished persisting bucket cache to file, renaming 2024-11-24T16:45:22,782 INFO [RS:0;e335232f7491:33045 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-24T16:45:22,782 INFO [RS:0;e335232f7491:33045 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-24T16:45:22,782 INFO [RS:2;e335232f7491:41507 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-24T16:45:22,782 INFO [RS:0;e335232f7491:33045 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-24T16:45:22,782 INFO [RS:2;e335232f7491:41507 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-24T16:45:22,782 INFO [RS:0;e335232f7491:33045 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-11-24T16:45:22,782 INFO [RS:2;e335232f7491:41507 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-24T16:45:22,782 INFO [RS:2;e335232f7491:41507 {}] regionserver.HRegionServer(3091): Received CLOSE for edbfb0487dd5f5fb1ea5f080d86356e0 2024-11-24T16:45:22,782 INFO [RS:0;e335232f7491:33045 {}] regionserver.HRegionServer(1321): Waiting on 1 regions to close 2024-11-24T16:45:22,782 DEBUG [RS:0;e335232f7491:33045 {}] regionserver.HRegionServer(1325): Online Regions={1588230740=hbase:meta,,1.1588230740} 2024-11-24T16:45:22,782 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-24T16:45:22,782 INFO [RS_CLOSE_META-regionserver/e335232f7491:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1662): Closing region hbase:meta,,1.1588230740 during a graceful stop, and cache persistence is on, so setting evict on close to false. 2024-11-24T16:45:22,782 DEBUG [RS:0;e335232f7491:33045 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-24T16:45:22,782 DEBUG [RS_CLOSE_META-regionserver/e335232f7491:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-24T16:45:22,782 INFO [RS:2;e335232f7491:41507 {}] regionserver.HRegionServer(959): stopping server e335232f7491,41507,1732466722156 2024-11-24T16:45:22,782 INFO [RS_CLOSE_META-regionserver/e335232f7491:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-24T16:45:22,783 INFO [RS:2;e335232f7491:41507 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-24T16:45:22,783 DEBUG [RS_CLOSE_META-regionserver/e335232f7491:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-24T16:45:22,783 DEBUG [RS_CLOSE_META-regionserver/e335232f7491:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-24T16:45:22,783 INFO [RS:2;e335232f7491:41507 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:2;e335232f7491:41507. 2024-11-24T16:45:22,783 DEBUG [RS_CLOSE_META-regionserver/e335232f7491:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-24T16:45:22,783 INFO [RS_CLOSE_REGION-regionserver/e335232f7491:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1662): Closing region testBlockEvictionOnGracefulStop1732466719962,,1732466719966.edbfb0487dd5f5fb1ea5f080d86356e0. during a graceful stop, and cache persistence is on, so setting evict on close to false. 2024-11-24T16:45:22,783 DEBUG [RS:2;e335232f7491:41507 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-24T16:45:22,783 DEBUG [RS_CLOSE_REGION-regionserver/e335232f7491:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing edbfb0487dd5f5fb1ea5f080d86356e0, disabling compactions & flushes 2024-11-24T16:45:22,783 DEBUG [RS:2;e335232f7491:41507 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T16:45:22,783 INFO [RS_CLOSE_META-regionserver/e335232f7491:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=1.21 KB heapSize=2.80 KB 2024-11-24T16:45:22,783 INFO [RS_CLOSE_REGION-regionserver/e335232f7491:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testBlockEvictionOnGracefulStop1732466719962,,1732466719966.edbfb0487dd5f5fb1ea5f080d86356e0. 2024-11-24T16:45:22,783 INFO [RS:2;e335232f7491:41507 {}] regionserver.HRegionServer(1321): Waiting on 1 regions to close 2024-11-24T16:45:22,783 DEBUG [RS_CLOSE_REGION-regionserver/e335232f7491:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testBlockEvictionOnGracefulStop1732466719962,,1732466719966.edbfb0487dd5f5fb1ea5f080d86356e0. 2024-11-24T16:45:22,783 DEBUG [RS:2;e335232f7491:41507 {}] regionserver.HRegionServer(1325): Online Regions={edbfb0487dd5f5fb1ea5f080d86356e0=testBlockEvictionOnGracefulStop1732466719962,,1732466719966.edbfb0487dd5f5fb1ea5f080d86356e0.} 2024-11-24T16:45:22,783 DEBUG [RS_CLOSE_REGION-regionserver/e335232f7491:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testBlockEvictionOnGracefulStop1732466719962,,1732466719966.edbfb0487dd5f5fb1ea5f080d86356e0. after waiting 0 ms 2024-11-24T16:45:22,783 DEBUG [RS_CLOSE_REGION-regionserver/e335232f7491:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testBlockEvictionOnGracefulStop1732466719962,,1732466719966.edbfb0487dd5f5fb1ea5f080d86356e0. 2024-11-24T16:45:22,783 DEBUG [RS:2;e335232f7491:41507 {}] regionserver.HRegionServer(1351): Waiting on edbfb0487dd5f5fb1ea5f080d86356e0 2024-11-24T16:45:22,785 DEBUG [StoreFileCloser-edbfb0487dd5f5fb1ea5f080d86356e0-family-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9797cf4c-7cf0-a873-8ba7-f30267b4a2f7/39f4fdd4-f188-1d18-4286-5b0c4dc27cad/data/default/testBlockEvictionOnGracefulStop1732466719962/edbfb0487dd5f5fb1ea5f080d86356e0/family/1ebd1eb39df242bbae9f0d38b310e39f 2024-11-24T16:45:22,790 DEBUG [RS_CLOSE_REGION-regionserver/e335232f7491:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9797cf4c-7cf0-a873-8ba7-f30267b4a2f7/39f4fdd4-f188-1d18-4286-5b0c4dc27cad/data/default/testBlockEvictionOnGracefulStop1732466719962/edbfb0487dd5f5fb1ea5f080d86356e0/recovered.edits/12.seqid, newMaxSeqId=12, maxSeqId=9 2024-11-24T16:45:22,791 INFO [RS_CLOSE_REGION-regionserver/e335232f7491:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testBlockEvictionOnGracefulStop1732466719962,,1732466719966.edbfb0487dd5f5fb1ea5f080d86356e0. 2024-11-24T16:45:22,791 DEBUG [RS_CLOSE_REGION-regionserver/e335232f7491:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for edbfb0487dd5f5fb1ea5f080d86356e0: Waiting for close lock at 1732466722783Running coprocessor pre-close hooks at 1732466722783Disabling compacts and flushes for region at 1732466722783Disabling writes for close at 1732466722783Writing region close event to WAL at 1732466722785 (+2 ms)Running coprocessor post-close hooks at 1732466722791 (+6 ms)Closed at 1732466722791 2024-11-24T16:45:22,791 DEBUG [RS_CLOSE_REGION-regionserver/e335232f7491:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testBlockEvictionOnGracefulStop1732466719962,,1732466719966.edbfb0487dd5f5fb1ea5f080d86356e0. 2024-11-24T16:45:22,793 DEBUG [RS_CLOSE_META-regionserver/e335232f7491:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFile$WriterFactory(308): Unable to set drop behind on bbcbd97999c7477abf28fc0d4e935ff4 2024-11-24T16:45:22,803 INFO [regionserver/e335232f7491:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-24T16:45:22,807 DEBUG [RS_CLOSE_META-regionserver/e335232f7491:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9797cf4c-7cf0-a873-8ba7-f30267b4a2f7/39f4fdd4-f188-1d18-4286-5b0c4dc27cad/data/hbase/meta/1588230740/.tmp/info/bbcbd97999c7477abf28fc0d4e935ff4 is 209, key is testBlockEvictionOnGracefulStop1732466719962,,1732466719966.edbfb0487dd5f5fb1ea5f080d86356e0./info:regioninfo/1732466722711/Put/seqid=0 2024-11-24T16:45:22,808 INFO [RS_CLOSE_META-regionserver/e335232f7491:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.21 KB at sequenceid=19 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9797cf4c-7cf0-a873-8ba7-f30267b4a2f7/39f4fdd4-f188-1d18-4286-5b0c4dc27cad/data/hbase/meta/1588230740/.tmp/info/bbcbd97999c7477abf28fc0d4e935ff4 2024-11-24T16:45:22,810 DEBUG [RS_CLOSE_META-regionserver/e335232f7491:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9797cf4c-7cf0-a873-8ba7-f30267b4a2f7/39f4fdd4-f188-1d18-4286-5b0c4dc27cad/data/hbase/meta/1588230740/.tmp/info/bbcbd97999c7477abf28fc0d4e935ff4 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9797cf4c-7cf0-a873-8ba7-f30267b4a2f7/39f4fdd4-f188-1d18-4286-5b0c4dc27cad/data/hbase/meta/1588230740/info/bbcbd97999c7477abf28fc0d4e935ff4 2024-11-24T16:45:22,811 DEBUG [RS_CLOSE_META-regionserver/e335232f7491:0-0 {event_type=M_RS_CLOSE_META}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9797cf4c-7cf0-a873-8ba7-f30267b4a2f7/39f4fdd4-f188-1d18-4286-5b0c4dc27cad/data/hbase/meta/1588230740/info/bbcbd97999c7477abf28fc0d4e935ff4, delay=939 ms 2024-11-24T16:45:22,812 INFO [RS_CLOSE_META-regionserver/e335232f7491:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9797cf4c-7cf0-a873-8ba7-f30267b4a2f7/39f4fdd4-f188-1d18-4286-5b0c4dc27cad/data/hbase/meta/1588230740/info/bbcbd97999c7477abf28fc0d4e935ff4, entries=8, sequenceid=19, filesize=6.6 K 2024-11-24T16:45:22,813 INFO [RS_CLOSE_META-regionserver/e335232f7491:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~1.21 KB/1238, heapSize ~2.03 KB/2080, currentSize=0 B/0 for 1588230740 in 30ms, sequenceid=19, compaction requested=false 2024-11-24T16:45:22,814 DEBUG [StoreFileCloser-1588230740-info-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9797cf4c-7cf0-a873-8ba7-f30267b4a2f7/39f4fdd4-f188-1d18-4286-5b0c4dc27cad/data/hbase/meta/1588230740/info/971b3bd28c36406c9e7e88e8e2172769 2024-11-24T16:45:22,814 DEBUG [StoreFileCloser-1588230740-info-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9797cf4c-7cf0-a873-8ba7-f30267b4a2f7/39f4fdd4-f188-1d18-4286-5b0c4dc27cad/data/hbase/meta/1588230740/info/bbcbd97999c7477abf28fc0d4e935ff4 2024-11-24T16:45:22,814 DEBUG [StoreFileCloser-1588230740-ns-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9797cf4c-7cf0-a873-8ba7-f30267b4a2f7/39f4fdd4-f188-1d18-4286-5b0c4dc27cad/data/hbase/meta/1588230740/ns/c1c6018341f54195884d54ad8cd64f1d 2024-11-24T16:45:22,815 DEBUG [StoreFileCloser-1588230740-table-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9797cf4c-7cf0-a873-8ba7-f30267b4a2f7/39f4fdd4-f188-1d18-4286-5b0c4dc27cad/data/hbase/meta/1588230740/table/6ee5d4979983472db67dd250d09f8b2a 2024-11-24T16:45:22,819 DEBUG [RS_CLOSE_META-regionserver/e335232f7491:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9797cf4c-7cf0-a873-8ba7-f30267b4a2f7/39f4fdd4-f188-1d18-4286-5b0c4dc27cad/data/hbase/meta/1588230740/recovered.edits/22.seqid, newMaxSeqId=22, maxSeqId=14 2024-11-24T16:45:22,819 DEBUG [RS_CLOSE_META-regionserver/e335232f7491:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-24T16:45:22,820 INFO [RS_CLOSE_META-regionserver/e335232f7491:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-24T16:45:22,820 DEBUG [RS_CLOSE_META-regionserver/e335232f7491:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1732466722782Running coprocessor pre-close hooks at 1732466722782Disabling compacts and flushes for region at 1732466722782Disabling writes for close at 1732466722783 (+1 ms)Obtaining lock to block concurrent updates at 1732466722783Preparing flush snapshotting stores in 1588230740 at 1732466722783Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=1238, getHeapSize=2800, getOffHeapSize=0, getCellsCount=8 at 1732466722783Flushing stores of hbase:meta,,1.1588230740 at 1732466722784 (+1 ms)Flushing 1588230740/info: creating writer at 1732466722784Flushing 1588230740/info: appending metadata at 1732466722807 (+23 ms)Flushing 1588230740/info: closing flushed file at 1732466722807Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@48a7c7cc: reopening flushed file at 1732466722810 (+3 ms)Finished flush of dataSize ~1.21 KB/1238, heapSize ~2.03 KB/2080, currentSize=0 B/0 for 1588230740 in 30ms, sequenceid=19, compaction requested=false at 1732466722813 (+3 ms)Writing region close event to WAL at 1732466722815 (+2 ms)Running coprocessor post-close hooks at 1732466722819 (+4 ms)Closed at 1732466722820 (+1 ms) 2024-11-24T16:45:22,820 DEBUG [RS_CLOSE_META-regionserver/e335232f7491:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-11-24T16:45:22,828 INFO [regionserver/e335232f7491:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-24T16:45:22,983 INFO [RS:0;e335232f7491:33045 {}] regionserver.HRegionServer(976): stopping server e335232f7491,33045,1732466718984; all regions closed. 2024-11-24T16:45:22,983 INFO [RS:2;e335232f7491:41507 {}] regionserver.HRegionServer(976): stopping server e335232f7491,41507,1732466722156; all regions closed. 2024-11-24T16:45:22,985 DEBUG [RS:2;e335232f7491:41507 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9797cf4c-7cf0-a873-8ba7-f30267b4a2f7/39f4fdd4-f188-1d18-4286-5b0c4dc27cad/oldWALs 2024-11-24T16:45:22,985 INFO [RS:2;e335232f7491:41507 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL e335232f7491%2C41507%2C1732466722156:(num 1732466722368) 2024-11-24T16:45:22,985 DEBUG [RS:2;e335232f7491:41507 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T16:45:22,985 INFO [RS:2;e335232f7491:41507 {}] regionserver.LeaseManager(133): Closed leases 2024-11-24T16:45:22,985 INFO [RS:2;e335232f7491:41507 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-24T16:45:22,985 DEBUG [RS:0;e335232f7491:33045 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9797cf4c-7cf0-a873-8ba7-f30267b4a2f7/39f4fdd4-f188-1d18-4286-5b0c4dc27cad/oldWALs 2024-11-24T16:45:22,986 INFO [RS:0;e335232f7491:33045 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL e335232f7491%2C33045%2C1732466718984.meta:.meta(num 1732466722196) 2024-11-24T16:45:22,986 INFO [RS:2;e335232f7491:41507 {}] hbase.ChoreService(370): Chore service for: regionserver/e335232f7491:0 had [ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS] on shutdown 2024-11-24T16:45:22,986 INFO [RS:2;e335232f7491:41507 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-24T16:45:22,986 INFO [RS:2;e335232f7491:41507 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-24T16:45:22,986 INFO [regionserver/e335232f7491:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-24T16:45:22,986 INFO [RS:2;e335232f7491:41507 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-24T16:45:22,986 INFO [RS:2;e335232f7491:41507 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-24T16:45:22,986 INFO [RS:2;e335232f7491:41507 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:41507 2024-11-24T16:45:22,988 DEBUG [RS:0;e335232f7491:33045 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9797cf4c-7cf0-a873-8ba7-f30267b4a2f7/39f4fdd4-f188-1d18-4286-5b0c4dc27cad/oldWALs 2024-11-24T16:45:22,988 INFO [RS:0;e335232f7491:33045 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL e335232f7491%2C33045%2C1732466718984:(num 1732466719535) 2024-11-24T16:45:22,988 DEBUG [RS:0;e335232f7491:33045 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T16:45:22,988 INFO [RS:0;e335232f7491:33045 {}] regionserver.LeaseManager(133): Closed leases 2024-11-24T16:45:22,988 INFO [RS:0;e335232f7491:33045 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-24T16:45:22,988 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36357-0x1014ade23570000, quorum=127.0.0.1:63542, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-24T16:45:22,988 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41507-0x1014ade23570003, quorum=127.0.0.1:63542, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/e335232f7491,41507,1732466722156 2024-11-24T16:45:22,988 INFO [RS:2;e335232f7491:41507 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-24T16:45:22,988 INFO [RS:0;e335232f7491:33045 {}] hbase.ChoreService(370): Chore service for: regionserver/e335232f7491:0 had [ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-11-24T16:45:22,988 INFO [RS:0;e335232f7491:33045 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-24T16:45:22,988 INFO [regionserver/e335232f7491:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-24T16:45:22,989 INFO [RS:0;e335232f7491:33045 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:33045 2024-11-24T16:45:22,990 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [e335232f7491,41507,1732466722156] 2024-11-24T16:45:22,991 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36357-0x1014ade23570000, quorum=127.0.0.1:63542, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-24T16:45:22,991 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33045-0x1014ade23570001, quorum=127.0.0.1:63542, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/e335232f7491,33045,1732466718984 2024-11-24T16:45:22,991 INFO [RS:0;e335232f7491:33045 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-24T16:45:22,991 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/e335232f7491,41507,1732466722156 already deleted, retry=false 2024-11-24T16:45:22,992 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; e335232f7491,41507,1732466722156 expired; onlineServers=1 2024-11-24T16:45:22,996 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [e335232f7491,33045,1732466718984] 2024-11-24T16:45:22,997 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/e335232f7491,33045,1732466718984 already deleted, retry=false 2024-11-24T16:45:22,997 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; e335232f7491,33045,1732466718984 expired; onlineServers=0 2024-11-24T16:45:22,997 INFO [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master 'e335232f7491,36357,1732466718936' ***** 2024-11-24T16:45:22,997 INFO [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0 2024-11-24T16:45:22,997 INFO [M:0;e335232f7491:36357 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-24T16:45:22,998 INFO [M:0;e335232f7491:36357 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-24T16:45:22,998 DEBUG [M:0;e335232f7491:36357 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2024-11-24T16:45:22,998 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2024-11-24T16:45:22,998 DEBUG [M:0;e335232f7491:36357 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2024-11-24T16:45:22,998 DEBUG [master/e335232f7491:0:becomeActiveMaster-HFileCleaner.small.0-1732466719265 {}] cleaner.HFileCleaner(306): Exit Thread[master/e335232f7491:0:becomeActiveMaster-HFileCleaner.small.0-1732466719265,5,FailOnTimeoutGroup] 2024-11-24T16:45:22,998 DEBUG [master/e335232f7491:0:becomeActiveMaster-HFileCleaner.large.0-1732466719264 {}] cleaner.HFileCleaner(306): Exit Thread[master/e335232f7491:0:becomeActiveMaster-HFileCleaner.large.0-1732466719264,5,FailOnTimeoutGroup] 2024-11-24T16:45:22,998 INFO [M:0;e335232f7491:36357 {}] hbase.ChoreService(370): Chore service for: master/e335232f7491:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown 2024-11-24T16:45:22,998 INFO [M:0;e335232f7491:36357 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-24T16:45:22,998 DEBUG [M:0;e335232f7491:36357 {}] master.HMaster(1795): Stopping service threads 2024-11-24T16:45:22,998 INFO [M:0;e335232f7491:36357 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2024-11-24T16:45:22,998 INFO [M:0;e335232f7491:36357 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-24T16:45:22,999 INFO [M:0;e335232f7491:36357 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-24T16:45:22,999 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2024-11-24T16:45:22,999 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36357-0x1014ade23570000, quorum=127.0.0.1:63542, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-11-24T16:45:22,999 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36357-0x1014ade23570000, quorum=127.0.0.1:63542, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T16:45:22,999 DEBUG [M:0;e335232f7491:36357 {}] zookeeper.ZKUtil(347): master:36357-0x1014ade23570000, quorum=127.0.0.1:63542, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error) 2024-11-24T16:45:22,999 WARN [M:0;e335232f7491:36357 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null 2024-11-24T16:45:22,999 INFO [M:0;e335232f7491:36357 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9797cf4c-7cf0-a873-8ba7-f30267b4a2f7/39f4fdd4-f188-1d18-4286-5b0c4dc27cad/.lastflushedseqids 2024-11-24T16:45:23,003 INFO [M:0;e335232f7491:36357 {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-24T16:45:23,003 INFO [M:0;e335232f7491:36357 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2024-11-24T16:45:23,004 DEBUG [M:0;e335232f7491:36357 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-24T16:45:23,004 INFO [M:0;e335232f7491:36357 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-24T16:45:23,004 DEBUG [M:0;e335232f7491:36357 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-24T16:45:23,004 DEBUG [M:0;e335232f7491:36357 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-24T16:45:23,004 DEBUG [M:0;e335232f7491:36357 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-24T16:45:23,004 INFO [M:0;e335232f7491:36357 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=39.49 KB heapSize=49.88 KB 2024-11-24T16:45:23,014 DEBUG [M:0;e335232f7491:36357 {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 7f53bb3f0b094a40b636262da90de36a 2024-11-24T16:45:23,028 DEBUG [M:0;e335232f7491:36357 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9797cf4c-7cf0-a873-8ba7-f30267b4a2f7/39f4fdd4-f188-1d18-4286-5b0c4dc27cad/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/7f53bb3f0b094a40b636262da90de36a is 82, key is hbase:meta,,1/info:regioninfo/1732466722261/Put/seqid=0 2024-11-24T16:45:23,029 INFO [M:0;e335232f7491:36357 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1008 B at sequenceid=102 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9797cf4c-7cf0-a873-8ba7-f30267b4a2f7/39f4fdd4-f188-1d18-4286-5b0c4dc27cad/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/7f53bb3f0b094a40b636262da90de36a 2024-11-24T16:45:23,037 DEBUG [M:0;e335232f7491:36357 {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 4a1b85c8c10c4fc4b5790b856fd394e7 2024-11-24T16:45:23,051 DEBUG [M:0;e335232f7491:36357 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9797cf4c-7cf0-a873-8ba7-f30267b4a2f7/39f4fdd4-f188-1d18-4286-5b0c4dc27cad/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/4a1b85c8c10c4fc4b5790b856fd394e7 is 781, key is \x00\x00\x00\x00\x00\x00\x00\x04/proc:d/1732466720346/Put/seqid=0 2024-11-24T16:45:23,052 INFO [M:0;e335232f7491:36357 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.26 KB at sequenceid=102 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9797cf4c-7cf0-a873-8ba7-f30267b4a2f7/39f4fdd4-f188-1d18-4286-5b0c4dc27cad/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/4a1b85c8c10c4fc4b5790b856fd394e7 2024-11-24T16:45:23,059 DEBUG [M:0;e335232f7491:36357 {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 67c6d93a3dc34ffeb3196407aff22afa 2024-11-24T16:45:23,073 DEBUG [M:0;e335232f7491:36357 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9797cf4c-7cf0-a873-8ba7-f30267b4a2f7/39f4fdd4-f188-1d18-4286-5b0c4dc27cad/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/67c6d93a3dc34ffeb3196407aff22afa is 69, key is e335232f7491,33045,1732466718984/rs:state/1732466719376/Put/seqid=0 2024-11-24T16:45:23,074 INFO [M:0;e335232f7491:36357 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=249 B at sequenceid=102 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9797cf4c-7cf0-a873-8ba7-f30267b4a2f7/39f4fdd4-f188-1d18-4286-5b0c4dc27cad/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/67c6d93a3dc34ffeb3196407aff22afa 2024-11-24T16:45:23,076 INFO [M:0;e335232f7491:36357 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 67c6d93a3dc34ffeb3196407aff22afa 2024-11-24T16:45:23,076 DEBUG [M:0;e335232f7491:36357 {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9797cf4c-7cf0-a873-8ba7-f30267b4a2f7/39f4fdd4-f188-1d18-4286-5b0c4dc27cad/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/7f53bb3f0b094a40b636262da90de36a as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9797cf4c-7cf0-a873-8ba7-f30267b4a2f7/39f4fdd4-f188-1d18-4286-5b0c4dc27cad/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/7f53bb3f0b094a40b636262da90de36a 2024-11-24T16:45:23,078 INFO [M:0;e335232f7491:36357 {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9797cf4c-7cf0-a873-8ba7-f30267b4a2f7/39f4fdd4-f188-1d18-4286-5b0c4dc27cad/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/7f53bb3f0b094a40b636262da90de36a, entries=14, sequenceid=102, filesize=5.9 K 2024-11-24T16:45:23,078 DEBUG [M:0;e335232f7491:36357 {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9797cf4c-7cf0-a873-8ba7-f30267b4a2f7/39f4fdd4-f188-1d18-4286-5b0c4dc27cad/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/4a1b85c8c10c4fc4b5790b856fd394e7 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9797cf4c-7cf0-a873-8ba7-f30267b4a2f7/39f4fdd4-f188-1d18-4286-5b0c4dc27cad/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/4a1b85c8c10c4fc4b5790b856fd394e7 2024-11-24T16:45:23,080 INFO [M:0;e335232f7491:36357 {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9797cf4c-7cf0-a873-8ba7-f30267b4a2f7/39f4fdd4-f188-1d18-4286-5b0c4dc27cad/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/4a1b85c8c10c4fc4b5790b856fd394e7, entries=11, sequenceid=102, filesize=6.5 K 2024-11-24T16:45:23,080 DEBUG [M:0;e335232f7491:36357 {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9797cf4c-7cf0-a873-8ba7-f30267b4a2f7/39f4fdd4-f188-1d18-4286-5b0c4dc27cad/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/67c6d93a3dc34ffeb3196407aff22afa as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9797cf4c-7cf0-a873-8ba7-f30267b4a2f7/39f4fdd4-f188-1d18-4286-5b0c4dc27cad/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/67c6d93a3dc34ffeb3196407aff22afa 2024-11-24T16:45:23,082 INFO [M:0;e335232f7491:36357 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 67c6d93a3dc34ffeb3196407aff22afa 2024-11-24T16:45:23,083 INFO [M:0;e335232f7491:36357 {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9797cf4c-7cf0-a873-8ba7-f30267b4a2f7/39f4fdd4-f188-1d18-4286-5b0c4dc27cad/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/67c6d93a3dc34ffeb3196407aff22afa, entries=3, sequenceid=102, filesize=5.3 K 2024-11-24T16:45:23,083 INFO [M:0;e335232f7491:36357 {}] regionserver.HRegion(3140): Finished flush of dataSize ~39.49 KB/40436, heapSize ~49.59 KB/50776, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 79ms, sequenceid=102, compaction requested=false 2024-11-24T16:45:23,085 INFO [M:0;e335232f7491:36357 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-24T16:45:23,085 DEBUG [M:0;e335232f7491:36357 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1732466723003Disabling compacts and flushes for region at 1732466723003Disabling writes for close at 1732466723004 (+1 ms)Obtaining lock to block concurrent updates at 1732466723004Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1732466723004Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=40436, getHeapSize=51016, getOffHeapSize=0, getCellsCount=127 at 1732466723004Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1732466723005 (+1 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1732466723005Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1732466723028 (+23 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1732466723028Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1732466723031 (+3 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1732466723051 (+20 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1732466723051Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1732466723053 (+2 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1732466723072 (+19 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1732466723072Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@683edbc6: reopening flushed file at 1732466723076 (+4 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@3686a10c: reopening flushed file at 1732466723078 (+2 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@56940447: reopening flushed file at 1732466723080 (+2 ms)Finished flush of dataSize ~39.49 KB/40436, heapSize ~49.59 KB/50776, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 79ms, sequenceid=102, compaction requested=false at 1732466723083 (+3 ms)Writing region close event to WAL at 1732466723085 (+2 ms)Closed at 1732466723085 2024-11-24T16:45:23,086 INFO [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-24T16:45:23,086 INFO [M:0;e335232f7491:36357 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down. 2024-11-24T16:45:23,086 INFO [M:0;e335232f7491:36357 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:36357 2024-11-24T16:45:23,087 INFO [M:0;e335232f7491:36357 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-24T16:45:23,091 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41507-0x1014ade23570003, quorum=127.0.0.1:63542, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-24T16:45:23,091 INFO [RS:2;e335232f7491:41507 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-24T16:45:23,091 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41507-0x1014ade23570003, quorum=127.0.0.1:63542, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-24T16:45:23,091 INFO [RS:2;e335232f7491:41507 {}] regionserver.HRegionServer(1031): Exiting; stopping=e335232f7491,41507,1732466722156; zookeeper connection closed. 2024-11-24T16:45:23,091 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@74ea0991 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@74ea0991 2024-11-24T16:45:23,096 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33045-0x1014ade23570001, quorum=127.0.0.1:63542, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-24T16:45:23,096 INFO [RS:0;e335232f7491:33045 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-24T16:45:23,096 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33045-0x1014ade23570001, quorum=127.0.0.1:63542, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-24T16:45:23,096 INFO [RS:0;e335232f7491:33045 {}] regionserver.HRegionServer(1031): Exiting; stopping=e335232f7491,33045,1732466718984; zookeeper connection closed. 2024-11-24T16:45:23,097 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@296feb9c {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@296feb9c 2024-11-24T16:45:23,097 INFO [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 3 regionserver(s) complete 2024-11-24T16:45:23,189 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36357-0x1014ade23570000, quorum=127.0.0.1:63542, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-24T16:45:23,189 INFO [M:0;e335232f7491:36357 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-24T16:45:23,189 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36357-0x1014ade23570000, quorum=127.0.0.1:63542, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-24T16:45:23,191 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers 2024-11-24T16:45:23,193 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down 2024-11-24T16:45:23,193 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-24T16:45:23,200 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: io.hfile.TestBlockEvictionOnRegionMovement#testBlockEvictionOnGracefulStop Thread=70 (was 49) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-7-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-9-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-9-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-8-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-8 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-5 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-7 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-9-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-7-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-6 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HMaster-EventLoopGroup-6-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-8-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HMaster-EventLoopGroup-6-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-7-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HMaster-EventLoopGroup-6-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-8-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=382 (was 346) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=612 (was 656), ProcessCount=11 (was 11), AvailableMemoryMB=6498 (was 6607)