2024-11-23 21:54:35,171 main DEBUG Apache Log4j Core 2.17.2 initializing configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@15a34df2 2024-11-23 21:54:35,188 main DEBUG Took 0.015295 seconds to load 1 plugins from package org.apache.hadoop.hbase.logging 2024-11-23 21:54:35,188 main DEBUG PluginManager 'Core' found 129 plugins 2024-11-23 21:54:35,189 main DEBUG PluginManager 'Level' found 0 plugins 2024-11-23 21:54:35,190 main DEBUG PluginManager 'Lookup' found 16 plugins 2024-11-23 21:54:35,191 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-23 21:54:35,197 main DEBUG PluginManager 'TypeConverter' found 26 plugins 2024-11-23 21:54:35,212 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.util.MBeans", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-23 21:54:35,213 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-23 21:54:35,214 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.logging.TestJul2Slf4j", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-23 21:54:35,214 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-23 21:54:35,215 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.zookeeper", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-23 21:54:35,215 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-23 21:54:35,216 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSinkAdapter", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-23 21:54:35,216 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-23 21:54:35,216 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSystemImpl", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-23 21:54:35,217 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-23 21:54:35,217 main DEBUG LoggerConfig$Builder(additivity="false", level="WARN", levelAndRefs="null", name="org.apache.directory", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-23 21:54:35,218 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-23 21:54:35,218 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.ipc.FailedServers", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-23 21:54:35,218 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-23 21:54:35,219 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsConfig", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-23 21:54:35,219 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-23 21:54:35,220 main DEBUG LoggerConfig$Builder(additivity="null", level="INFO", levelAndRefs="null", name="org.apache.hadoop.hbase.ScheduledChore", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-23 21:54:35,220 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-23 21:54:35,220 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.regionserver.RSRpcServices", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-23 21:54:35,221 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-23 21:54:35,221 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-23 21:54:35,221 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-23 21:54:35,222 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-23 21:54:35,222 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-23 21:54:35,222 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hbase.thirdparty.io.netty.channel", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-23 21:54:35,223 main DEBUG Building Plugin[name=root, class=org.apache.logging.log4j.core.config.LoggerConfig$RootLogger]. 2024-11-23 21:54:35,224 main DEBUG LoggerConfig$RootLogger$Builder(additivity="null", level="null", levelAndRefs="INFO,Console", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-23 21:54:35,225 main DEBUG Building Plugin[name=loggers, class=org.apache.logging.log4j.core.config.LoggersPlugin]. 2024-11-23 21:54:35,227 main DEBUG createLoggers(={org.apache.hadoop.metrics2.util.MBeans, org.apache.hadoop.hbase.logging.TestJul2Slf4j, org.apache.zookeeper, org.apache.hadoop.metrics2.impl.MetricsSinkAdapter, org.apache.hadoop.metrics2.impl.MetricsSystemImpl, org.apache.directory, org.apache.hadoop.hbase.ipc.FailedServers, org.apache.hadoop.metrics2.impl.MetricsConfig, org.apache.hadoop.hbase.ScheduledChore, org.apache.hadoop.hbase.regionserver.RSRpcServices, org.apache.hadoop, org.apache.hadoop.hbase, org.apache.hbase.thirdparty.io.netty.channel, root}) 2024-11-23 21:54:35,227 main DEBUG Building Plugin[name=layout, class=org.apache.logging.log4j.core.layout.PatternLayout]. 2024-11-23 21:54:35,228 main DEBUG PatternLayout$Builder(pattern="%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n", PatternSelector=null, Configuration(PropertiesConfig), Replace=null, charset="null", alwaysWriteExceptions="null", disableAnsi="null", noConsoleNoAnsi="null", header="null", footer="null") 2024-11-23 21:54:35,229 main DEBUG PluginManager 'Converter' found 47 plugins 2024-11-23 21:54:35,236 main DEBUG Building Plugin[name=appender, class=org.apache.hadoop.hbase.logging.HBaseTestAppender]. 2024-11-23 21:54:35,238 main DEBUG HBaseTestAppender$Builder(target="SYSTEM_ERR", maxSize="1G", bufferedIo="null", bufferSize="null", immediateFlush="null", ignoreExceptions="null", PatternLayout(%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n), name="Console", Configuration(PropertiesConfig), Filter=null, ={}) 2024-11-23 21:54:35,240 main DEBUG Starting HBaseTestOutputStreamManager SYSTEM_ERR 2024-11-23 21:54:35,240 main DEBUG Building Plugin[name=appenders, class=org.apache.logging.log4j.core.config.AppendersPlugin]. 2024-11-23 21:54:35,241 main DEBUG createAppenders(={Console}) 2024-11-23 21:54:35,242 main DEBUG Configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@15a34df2 initialized 2024-11-23 21:54:35,242 main DEBUG Starting configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@15a34df2 2024-11-23 21:54:35,242 main DEBUG Started configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@15a34df2 OK. 2024-11-23 21:54:35,243 main DEBUG Shutting down OutputStreamManager SYSTEM_OUT.false.false-1 2024-11-23 21:54:35,243 main DEBUG OutputStream closed 2024-11-23 21:54:35,243 main DEBUG Shut down OutputStreamManager SYSTEM_OUT.false.false-1, all resources released: true 2024-11-23 21:54:35,243 main DEBUG Appender DefaultConsole-1 stopped with status true 2024-11-23 21:54:35,244 main DEBUG Stopped org.apache.logging.log4j.core.config.DefaultConfiguration@4f2b503c OK 2024-11-23 21:54:35,313 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6 2024-11-23 21:54:35,315 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=StatusLogger 2024-11-23 21:54:35,316 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=ContextSelector 2024-11-23 21:54:35,317 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name= 2024-11-23 21:54:35,318 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.directory 2024-11-23 21:54:35,319 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSinkAdapter 2024-11-23 21:54:35,319 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.zookeeper 2024-11-23 21:54:35,319 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.logging.TestJul2Slf4j 2024-11-23 21:54:35,320 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSystemImpl 2024-11-23 21:54:35,320 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.util.MBeans 2024-11-23 21:54:35,321 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase 2024-11-23 21:54:35,321 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop 2024-11-23 21:54:35,322 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ipc.FailedServers 2024-11-23 21:54:35,322 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.regionserver.RSRpcServices 2024-11-23 21:54:35,322 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsConfig 2024-11-23 21:54:35,323 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hbase.thirdparty.io.netty.channel 2024-11-23 21:54:35,323 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ScheduledChore 2024-11-23 21:54:35,324 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Appenders,name=Console 2024-11-23 21:54:35,326 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-23 21:54:35,327 main DEBUG Reconfiguration complete for context[name=1dbd16a6] at URI jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-logging/target/hbase-logging-4.0.0-alpha-1-SNAPSHOT-tests.jar!/log4j2.properties (org.apache.logging.log4j.core.LoggerContext@31c7528f) with optional ClassLoader: null 2024-11-23 21:54:35,327 main DEBUG Shutdown hook enabled. Registering a new one. 2024-11-23 21:54:35,328 main DEBUG LoggerContext[name=1dbd16a6, org.apache.logging.log4j.core.LoggerContext@31c7528f] started OK. 2024-11-23T21:54:35,340 INFO [main {}] hbase.HBaseClassTestRule(94): Test class org.apache.hadoop.hbase.io.hfile.TestBlockEvictionOnRegionMovement timeout: 13 mins 2024-11-23 21:54:35,343 main DEBUG AsyncLogger.ThreadNameStrategy=UNCACHED (user specified null, default is UNCACHED) 2024-11-23 21:54:35,343 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-23T21:54:35,590 DEBUG [main {}] hbase.HBaseTestingUtil(323): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/93be71cd-4dcf-2478-daf8-a2306a879141 2024-11-23T21:54:35,612 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: io.hfile.TestBlockEvictionOnRegionMovement#testBlockEvictionOnRegionMove Thread=12, OpenFileDescriptor=287, MaxFileDescriptor=1048576, SystemLoadAverage=445, ProcessCount=11, AvailableMemoryMB=8883 2024-11-23T21:54:35,707 WARN [Time-limited test {}] util.NativeCodeLoader(60): Unable to load native-hadoop library for your platform... using builtin-java classes where applicable 2024-11-23T21:54:35,791 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-23T21:54:35,807 INFO [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/93be71cd-4dcf-2478-daf8-a2306a879141/cluster_ce617a3e-572d-da39-b9d3-58198448f6f2, deleteOnExit=true 2024-11-23T21:54:35,939 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/93be71cd-4dcf-2478-daf8-a2306a879141/cluster_ce617a3e-572d-da39-b9d3-58198448f6f2/zookeeper_0, clientPort=60787, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/93be71cd-4dcf-2478-daf8-a2306a879141/cluster_ce617a3e-572d-da39-b9d3-58198448f6f2/zookeeper_0/version-2, dataDirSize=457 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/93be71cd-4dcf-2478-daf8-a2306a879141/cluster_ce617a3e-572d-da39-b9d3-58198448f6f2/zookeeper_0/version-2, dataLogSize=457 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0 2024-11-23T21:54:35,950 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=60787 2024-11-23T21:54:35,950 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-23T21:54:36,021 INFO [Time-limited test {}] util.FSUtils(489): Created version file at /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/93be71cd-4dcf-2478-daf8-a2306a879141/e0862929-e242-795d-1e0b-fe7766981a5b with version=8 2024-11-23T21:54:36,021 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1139): Setting hbase.fs.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/93be71cd-4dcf-2478-daf8-a2306a879141/e0862929-e242-795d-1e0b-fe7766981a5b/hbase-staging 2024-11-23T21:54:36,110 DEBUG [Time-limited test {}] channel.MultithreadEventLoopGroup(44): -Dio.netty.eventLoopThreads: 16 2024-11-23T21:54:36,355 INFO [Time-limited test {}] client.ConnectionUtils(128): master/adda16083778:0 server-side Connection retries=45 2024-11-23T21:54:36,364 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-23T21:54:36,365 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-23T21:54:36,375 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-23T21:54:36,375 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-23T21:54:36,376 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-23T21:54:36,518 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService 2024-11-23T21:54:36,598 WARN [Time-limited test {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-23T21:54:36,644 INFO [Time-limited test {}] metrics.MetricRegistriesLoader(60): Loaded MetricRegistries class org.apache.hadoop.hbase.metrics.impl.MetricRegistriesImpl 2024-11-23T21:54:36,654 DEBUG [Time-limited test {}] util.ClassSize(228): Using Unsafe to estimate memory layout 2024-11-23T21:54:36,657 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-23T21:54:36,691 DEBUG [Time-limited test {}] channel.DefaultChannelId(84): -Dio.netty.processId: 94794 (auto-detected) 2024-11-23T21:54:36,692 DEBUG [Time-limited test {}] channel.DefaultChannelId(106): -Dio.netty.machineId: 02:42:ac:ff:fe:11:00:02 (auto-detected) 2024-11-23T21:54:36,715 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:36037 2024-11-23T21:54:36,740 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:36037 connecting to ZooKeeper ensemble=127.0.0.1:60787 2024-11-23T21:54:36,885 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:360370x0, quorum=127.0.0.1:60787, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-23T21:54:36,887 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:36037-0x1016afa97d20000 connected 2024-11-23T21:54:36,989 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-23T21:54:36,991 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-23T21:54:37,002 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:36037-0x1016afa97d20000, quorum=127.0.0.1:60787, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-23T21:54:37,005 INFO [Time-limited test {}] master.HMaster(525): hbase.rootdir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/93be71cd-4dcf-2478-daf8-a2306a879141/e0862929-e242-795d-1e0b-fe7766981a5b, hbase.cluster.distributed=false 2024-11-23T21:54:37,028 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:36037-0x1016afa97d20000, quorum=127.0.0.1:60787, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-23T21:54:37,034 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=36037 2024-11-23T21:54:37,034 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=36037 2024-11-23T21:54:37,034 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=36037 2024-11-23T21:54:37,035 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=36037 2024-11-23T21:54:37,035 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=36037 2024-11-23T21:54:37,107 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/adda16083778:0 server-side Connection retries=45 2024-11-23T21:54:37,109 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-23T21:54:37,109 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-23T21:54:37,109 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-23T21:54:37,109 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-23T21:54:37,109 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-23T21:54:37,112 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-23T21:54:37,115 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-23T21:54:37,116 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:45163 2024-11-23T21:54:37,118 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:45163 connecting to ZooKeeper ensemble=127.0.0.1:60787 2024-11-23T21:54:37,118 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-23T21:54:37,119 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-23T21:54:37,158 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:451630x0, quorum=127.0.0.1:60787, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-23T21:54:37,159 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:45163-0x1016afa97d20001 connected 2024-11-23T21:54:37,159 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:45163-0x1016afa97d20001, quorum=127.0.0.1:60787, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-23T21:54:37,170 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-23T21:54:37,183 INFO [Time-limited test {}] bucket.FileIOEngine(99): Allocating cache 400 MB, on the path:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/93be71cd-4dcf-2478-daf8-a2306a879141/bucket.cache 2024-11-23T21:54:37,190 INFO [Time-limited test {}] bucket.BucketCache(364): Instantiating BucketCache with acceptableFactor: 0.95, minFactor: 0.85, extraFreeFactor: 0.1, singleFactor: 0.25, multiFactor: 0.5, memoryFactor: 0.25, useStrongRef: false 2024-11-23T21:54:37,192 INFO [Time-limited test {}] bucket.BucketCache(478): Starting BucketCachePersisterThread 2024-11-23T21:54:37,193 INFO [Time-limited test {}] bucket.BucketCachePersister(38): BucketCachePersister started with interval: 100 2024-11-23T21:54:37,195 INFO [Thread-10 {}] bucket.BucketCache(1530): Started retrieving bucket cache from file 2024-11-23T21:54:37,195 WARN [Thread-10 {}] bucket.BucketCache(1533): Persistence file missing! It's ok if it's first run after enabling persistent cache. 2024-11-23T21:54:37,201 INFO [Time-limited test {}] bucket.BucketCache(403): Started bucket cache; ioengine=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/93be71cd-4dcf-2478-daf8-a2306a879141/bucket.cache, capacity=400 MB, blockSize=64 KB, writerThreadNum=3, writerQLen=64, persistencePath=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/93be71cd-4dcf-2478-daf8-a2306a879141/bucket.persistence, bucketAllocator=org.apache.hadoop.hbase.io.hfile.bucket.BucketAllocator 2024-11-23T21:54:37,203 INFO [Thread-10 {}] bucket.BucketAllocator(343): Cache totalSize=418148352, buckets=199, bucket capacity=2101248=(4*525312)=(FEWEST_ITEMS_IN_BUCKET*(largest configured bucketcache size)) 2024-11-23T21:54:37,203 INFO [Thread-10 {}] bucket.BucketCache(413): Persistent bucket cache recovery from /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/93be71cd-4dcf-2478-daf8-a2306a879141/bucket.persistence is complete. 2024-11-23T21:54:37,207 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-23T21:54:37,210 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:45163-0x1016afa97d20001, quorum=127.0.0.1:60787, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-23T21:54:37,216 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:45163-0x1016afa97d20001, quorum=127.0.0.1:60787, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-23T21:54:37,216 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=45163 2024-11-23T21:54:37,217 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=45163 2024-11-23T21:54:37,217 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=45163 2024-11-23T21:54:37,218 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=45163 2024-11-23T21:54:37,218 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=45163 2024-11-23T21:54:37,238 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/adda16083778:0 server-side Connection retries=45 2024-11-23T21:54:37,238 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-23T21:54:37,238 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-23T21:54:37,239 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-23T21:54:37,239 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-23T21:54:37,239 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-23T21:54:37,240 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-23T21:54:37,240 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-23T21:54:37,241 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:42343 2024-11-23T21:54:37,245 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:42343 connecting to ZooKeeper ensemble=127.0.0.1:60787 2024-11-23T21:54:37,246 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-23T21:54:37,247 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-23T21:54:37,266 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:423430x0, quorum=127.0.0.1:60787, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-23T21:54:37,267 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:423430x0, quorum=127.0.0.1:60787, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-23T21:54:37,267 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:42343-0x1016afa97d20002 connected 2024-11-23T21:54:37,268 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-23T21:54:37,269 INFO [Time-limited test {}] bucket.FileIOEngine(99): Allocating cache 400 MB, on the path:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/93be71cd-4dcf-2478-daf8-a2306a879141/bucket.cache 2024-11-23T21:54:37,269 INFO [Time-limited test {}] bucket.BucketCache(364): Instantiating BucketCache with acceptableFactor: 0.95, minFactor: 0.85, extraFreeFactor: 0.1, singleFactor: 0.25, multiFactor: 0.5, memoryFactor: 0.25, useStrongRef: false 2024-11-23T21:54:37,270 INFO [Time-limited test {}] bucket.BucketCache(478): Starting BucketCachePersisterThread 2024-11-23T21:54:37,270 INFO [Time-limited test {}] bucket.BucketCachePersister(38): BucketCachePersister started with interval: 100 2024-11-23T21:54:37,270 INFO [Thread-11 {}] bucket.BucketCache(1530): Started retrieving bucket cache from file 2024-11-23T21:54:37,271 WARN [Thread-11 {}] bucket.BucketCache(1533): Persistence file missing! It's ok if it's first run after enabling persistent cache. 2024-11-23T21:54:37,271 INFO [Thread-11 {}] bucket.BucketAllocator(343): Cache totalSize=418148352, buckets=199, bucket capacity=2101248=(4*525312)=(FEWEST_ITEMS_IN_BUCKET*(largest configured bucketcache size)) 2024-11-23T21:54:37,272 INFO [Thread-11 {}] bucket.BucketCache(413): Persistent bucket cache recovery from /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/93be71cd-4dcf-2478-daf8-a2306a879141/bucket.persistence is complete. 2024-11-23T21:54:37,272 INFO [Time-limited test {}] bucket.BucketCache(403): Started bucket cache; ioengine=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/93be71cd-4dcf-2478-daf8-a2306a879141/bucket.cache, capacity=400 MB, blockSize=64 KB, writerThreadNum=3, writerQLen=64, persistencePath=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/93be71cd-4dcf-2478-daf8-a2306a879141/bucket.persistence, bucketAllocator=org.apache.hadoop.hbase.io.hfile.bucket.BucketAllocator 2024-11-23T21:54:37,274 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-23T21:54:37,276 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:42343-0x1016afa97d20002, quorum=127.0.0.1:60787, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-23T21:54:37,279 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:42343-0x1016afa97d20002, quorum=127.0.0.1:60787, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-23T21:54:37,283 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=42343 2024-11-23T21:54:37,284 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=42343 2024-11-23T21:54:37,284 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=42343 2024-11-23T21:54:37,285 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=42343 2024-11-23T21:54:37,286 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=42343 2024-11-23T21:54:37,302 DEBUG [M:0;adda16083778:36037 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;adda16083778:36037 2024-11-23T21:54:37,303 INFO [master/adda16083778:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/adda16083778,36037,1732398876186 2024-11-23T21:54:37,316 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42343-0x1016afa97d20002, quorum=127.0.0.1:60787, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-23T21:54:37,316 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36037-0x1016afa97d20000, quorum=127.0.0.1:60787, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-23T21:54:37,317 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45163-0x1016afa97d20001, quorum=127.0.0.1:60787, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-23T21:54:37,318 DEBUG [master/adda16083778:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:36037-0x1016afa97d20000, quorum=127.0.0.1:60787, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/adda16083778,36037,1732398876186 2024-11-23T21:54:37,349 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42343-0x1016afa97d20002, quorum=127.0.0.1:60787, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-23T21:54:37,349 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45163-0x1016afa97d20001, quorum=127.0.0.1:60787, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-23T21:54:37,349 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36037-0x1016afa97d20000, quorum=127.0.0.1:60787, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-23T21:54:37,349 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42343-0x1016afa97d20002, quorum=127.0.0.1:60787, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-23T21:54:37,350 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45163-0x1016afa97d20001, quorum=127.0.0.1:60787, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-23T21:54:37,351 DEBUG [master/adda16083778:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:36037-0x1016afa97d20000, quorum=127.0.0.1:60787, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-11-23T21:54:37,352 INFO [master/adda16083778:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/adda16083778,36037,1732398876186 from backup master directory 2024-11-23T21:54:37,366 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45163-0x1016afa97d20001, quorum=127.0.0.1:60787, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-23T21:54:37,366 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42343-0x1016afa97d20002, quorum=127.0.0.1:60787, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-23T21:54:37,366 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36037-0x1016afa97d20000, quorum=127.0.0.1:60787, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/adda16083778,36037,1732398876186 2024-11-23T21:54:37,366 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36037-0x1016afa97d20000, quorum=127.0.0.1:60787, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-23T21:54:37,367 WARN [master/adda16083778:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-23T21:54:37,368 INFO [master/adda16083778:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=adda16083778,36037,1732398876186 2024-11-23T21:54:37,370 INFO [master/adda16083778:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating data MemStoreChunkPool with chunk size 2 MB, max count 396, initial count 0 2024-11-23T21:54:37,373 INFO [master/adda16083778:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating index MemStoreChunkPool with chunk size 204.80 KB, max count 440, initial count 0 2024-11-23T21:54:37,381 DEBUG [master/adda16083778:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/93be71cd-4dcf-2478-daf8-a2306a879141/e0862929-e242-795d-1e0b-fe7766981a5b/hbase.id] with ID: 3cb83c07-7b6f-42b5-a954-50dac1670ae4 2024-11-23T21:54:37,381 DEBUG [master/adda16083778:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/93be71cd-4dcf-2478-daf8-a2306a879141/e0862929-e242-795d-1e0b-fe7766981a5b/.tmp/hbase.id 2024-11-23T21:54:37,386 DEBUG [master/adda16083778:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/93be71cd-4dcf-2478-daf8-a2306a879141/e0862929-e242-795d-1e0b-fe7766981a5b/.tmp/hbase.id]:[file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/93be71cd-4dcf-2478-daf8-a2306a879141/e0862929-e242-795d-1e0b-fe7766981a5b/hbase.id] 2024-11-23T21:54:37,418 DEBUG [master/adda16083778:0:becomeActiveMaster {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-23T21:54:37,424 INFO [master/adda16083778:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem. 2024-11-23T21:54:37,445 INFO [master/adda16083778:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 18ms. 2024-11-23T21:54:37,474 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45163-0x1016afa97d20001, quorum=127.0.0.1:60787, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-23T21:54:37,474 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42343-0x1016afa97d20002, quorum=127.0.0.1:60787, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-23T21:54:37,474 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36037-0x1016afa97d20000, quorum=127.0.0.1:60787, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-23T21:54:37,497 INFO [master/adda16083778:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-23T21:54:37,500 INFO [master/adda16083778:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-11-23T21:54:37,538 DEBUG [master/adda16083778:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(244): No decryptEncryptedDataEncryptionKey method in DFSClient, should be hadoop version with HDFS-12396 java.lang.NoSuchMethodException: org.apache.hadoop.hdfs.DFSClient.decryptEncryptedDataEncryptionKey(org.apache.hadoop.fs.FileEncryptionInfo) at java.lang.Class.getDeclaredMethod(Class.java:2675) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelperWithoutHDFS12396(FanOutOneBlockAsyncDFSOutputSaslHelper.java:183) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelper(FanOutOneBlockAsyncDFSOutputSaslHelper.java:242) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.(FanOutOneBlockAsyncDFSOutputSaslHelper.java:253) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.lang.Class.forName0(Native Method) ~[?:?] at java.lang.Class.forName(Class.java:375) ~[?:?] at org.apache.hadoop.hbase.wal.AsyncFSWALProvider.load(AsyncFSWALProvider.java:150) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.getProviderClass(WALFactory.java:174) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:262) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:231) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegion.create(MasterRegion.java:400) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegionFactory.create(MasterRegionFactory.java:135) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.finishActiveMasterInitialization(HMaster.java:1003) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.startActiveMasterManager(HMaster.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.lambda$run$0(HMaster.java:613) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.lambda$tracedRunnable$2(TraceUtil.java:155) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T21:54:37,544 INFO [master/adda16083778:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-23T21:54:37,597 INFO [master/adda16083778:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/93be71cd-4dcf-2478-daf8-a2306a879141/e0862929-e242-795d-1e0b-fe7766981a5b/MasterData/data/master/store 2024-11-23T21:54:37,623 INFO [master/adda16083778:0:becomeActiveMaster {}] throttle.StoreHotnessProtector(112): StoreHotnessProtector is disabled. Set hbase.region.store.parallel.put.limit > 0 to enable, which may help mitigate load under heavy write pressure. 2024-11-23T21:54:37,627 DEBUG [master/adda16083778:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-23T21:54:37,629 DEBUG [master/adda16083778:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-23T21:54:37,630 INFO [master/adda16083778:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-23T21:54:37,630 DEBUG [master/adda16083778:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-23T21:54:37,633 DEBUG [master/adda16083778:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-23T21:54:37,633 DEBUG [master/adda16083778:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-23T21:54:37,633 INFO [master/adda16083778:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-23T21:54:37,634 DEBUG [master/adda16083778:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1732398877629Disabling compacts and flushes for region at 1732398877629Disabling writes for close at 1732398877633 (+4 ms)Writing region close event to WAL at 1732398877633Closed at 1732398877633 2024-11-23T21:54:37,635 WARN [master/adda16083778:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/93be71cd-4dcf-2478-daf8-a2306a879141/e0862929-e242-795d-1e0b-fe7766981a5b/MasterData/data/master/store/.initializing 2024-11-23T21:54:37,635 DEBUG [master/adda16083778:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/93be71cd-4dcf-2478-daf8-a2306a879141/e0862929-e242-795d-1e0b-fe7766981a5b/MasterData/WALs/adda16083778,36037,1732398876186 2024-11-23T21:54:37,650 INFO [master/adda16083778:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-23T21:54:37,664 INFO [master/adda16083778:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=adda16083778%2C36037%2C1732398876186, suffix=, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/93be71cd-4dcf-2478-daf8-a2306a879141/e0862929-e242-795d-1e0b-fe7766981a5b/MasterData/WALs/adda16083778,36037,1732398876186, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/93be71cd-4dcf-2478-daf8-a2306a879141/e0862929-e242-795d-1e0b-fe7766981a5b/MasterData/oldWALs, maxLogs=10 2024-11-23T21:54:37,702 INFO [master/adda16083778:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/93be71cd-4dcf-2478-daf8-a2306a879141/e0862929-e242-795d-1e0b-fe7766981a5b/MasterData/WALs/adda16083778,36037,1732398876186/adda16083778%2C36037%2C1732398876186.1732398877668 2024-11-23T21:54:37,703 DEBUG [master/adda16083778:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-11-23T21:54:37,704 DEBUG [master/adda16083778:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-11-23T21:54:37,704 DEBUG [master/adda16083778:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-23T21:54:37,708 DEBUG [master/adda16083778:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-11-23T21:54:37,709 DEBUG [master/adda16083778:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-11-23T21:54:37,745 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-11-23T21:54:37,778 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-11-23T21:54:37,783 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T21:54:37,786 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-23T21:54:37,786 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-11-23T21:54:37,791 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-11-23T21:54:37,791 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T21:54:37,792 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-23T21:54:37,792 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-11-23T21:54:37,796 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-11-23T21:54:37,797 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T21:54:37,797 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-23T21:54:37,798 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-11-23T21:54:37,802 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-11-23T21:54:37,803 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T21:54:37,803 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-23T21:54:37,804 DEBUG [master/adda16083778:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682 2024-11-23T21:54:37,807 DEBUG [master/adda16083778:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/93be71cd-4dcf-2478-daf8-a2306a879141/e0862929-e242-795d-1e0b-fe7766981a5b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-23T21:54:37,808 DEBUG [master/adda16083778:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/93be71cd-4dcf-2478-daf8-a2306a879141/e0862929-e242-795d-1e0b-fe7766981a5b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-23T21:54:37,810 DEBUG [master/adda16083778:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682 2024-11-23T21:54:37,811 DEBUG [master/adda16083778:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682 2024-11-23T21:54:37,813 DEBUG [master/adda16083778:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-23T21:54:37,815 DEBUG [master/adda16083778:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-11-23T21:54:37,822 DEBUG [master/adda16083778:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/93be71cd-4dcf-2478-daf8-a2306a879141/e0862929-e242-795d-1e0b-fe7766981a5b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-23T21:54:37,823 INFO [master/adda16083778:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=66206842, jitterRate=-0.01344117522239685}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-23T21:54:37,830 DEBUG [master/adda16083778:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682: Writing region info on filesystem at 1732398877721Initializing all the Stores at 1732398877722 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732398877722Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732398877723 (+1 ms)Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732398877723Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732398877723Cleaning up temporary data from old regions at 1732398877811 (+88 ms)Region opened successfully at 1732398877830 (+19 ms) 2024-11-23T21:54:37,832 INFO [master/adda16083778:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-11-23T21:54:37,859 DEBUG [master/adda16083778:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@75d6bb36, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=adda16083778/172.17.0.2:0 2024-11-23T21:54:37,890 INFO [master/adda16083778:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating... 2024-11-23T21:54:37,904 INFO [master/adda16083778:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5 2024-11-23T21:54:37,904 INFO [master/adda16083778:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50 2024-11-23T21:54:37,908 INFO [master/adda16083778:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-23T21:54:37,908 INFO [master/adda16083778:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-23T21:54:37,915 INFO [master/adda16083778:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 5 msec 2024-11-23T21:54:37,915 INFO [master/adda16083778:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-11-23T21:54:37,944 INFO [master/adda16083778:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2024-11-23T21:54:37,954 DEBUG [master/adda16083778:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:36037-0x1016afa97d20000, quorum=127.0.0.1:60787, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error) 2024-11-23T21:54:38,007 DEBUG [master/adda16083778:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false 2024-11-23T21:54:38,011 INFO [master/adda16083778:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2024-11-23T21:54:38,014 DEBUG [master/adda16083778:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:36037-0x1016afa97d20000, quorum=127.0.0.1:60787, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error) 2024-11-23T21:54:38,024 DEBUG [master/adda16083778:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false 2024-11-23T21:54:38,026 INFO [master/adda16083778:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2024-11-23T21:54:38,031 DEBUG [master/adda16083778:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:36037-0x1016afa97d20000, quorum=127.0.0.1:60787, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error) 2024-11-23T21:54:38,040 DEBUG [master/adda16083778:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false 2024-11-23T21:54:38,043 DEBUG [master/adda16083778:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:36037-0x1016afa97d20000, quorum=127.0.0.1:60787, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error) 2024-11-23T21:54:38,049 DEBUG [master/adda16083778:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false 2024-11-23T21:54:38,068 DEBUG [master/adda16083778:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:36037-0x1016afa97d20000, quorum=127.0.0.1:60787, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error) 2024-11-23T21:54:38,080 DEBUG [master/adda16083778:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false 2024-11-23T21:54:38,099 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36037-0x1016afa97d20000, quorum=127.0.0.1:60787, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-23T21:54:38,099 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45163-0x1016afa97d20001, quorum=127.0.0.1:60787, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-23T21:54:38,099 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42343-0x1016afa97d20002, quorum=127.0.0.1:60787, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-23T21:54:38,099 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45163-0x1016afa97d20001, quorum=127.0.0.1:60787, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-23T21:54:38,099 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36037-0x1016afa97d20000, quorum=127.0.0.1:60787, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-23T21:54:38,099 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42343-0x1016afa97d20002, quorum=127.0.0.1:60787, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-23T21:54:38,102 INFO [master/adda16083778:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=adda16083778,36037,1732398876186, sessionid=0x1016afa97d20000, setting cluster-up flag (Was=false) 2024-11-23T21:54:38,124 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36037-0x1016afa97d20000, quorum=127.0.0.1:60787, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-23T21:54:38,124 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42343-0x1016afa97d20002, quorum=127.0.0.1:60787, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-23T21:54:38,124 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45163-0x1016afa97d20001, quorum=127.0.0.1:60787, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-23T21:54:38,149 DEBUG [master/adda16083778:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort 2024-11-23T21:54:38,152 DEBUG [master/adda16083778:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=adda16083778,36037,1732398876186 2024-11-23T21:54:38,174 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36037-0x1016afa97d20000, quorum=127.0.0.1:60787, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-23T21:54:38,174 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42343-0x1016afa97d20002, quorum=127.0.0.1:60787, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-23T21:54:38,174 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45163-0x1016afa97d20001, quorum=127.0.0.1:60787, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-23T21:54:38,199 DEBUG [master/adda16083778:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort 2024-11-23T21:54:38,201 DEBUG [master/adda16083778:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=adda16083778,36037,1732398876186 2024-11-23T21:54:38,208 INFO [master/adda16083778:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/93be71cd-4dcf-2478-daf8-a2306a879141/e0862929-e242-795d-1e0b-fe7766981a5b/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again 2024-11-23T21:54:38,276 DEBUG [master/adda16083778:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-23T21:54:38,288 INFO [master/adda16083778:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2 2024-11-23T21:54:38,291 INFO [RS:1;adda16083778:42343 {}] regionserver.HRegionServer(746): ClusterId : 3cb83c07-7b6f-42b5-a954-50dac1670ae4 2024-11-23T21:54:38,291 INFO [RS:0;adda16083778:45163 {}] regionserver.HRegionServer(746): ClusterId : 3cb83c07-7b6f-42b5-a954-50dac1670ae4 2024-11-23T21:54:38,294 DEBUG [RS:1;adda16083778:42343 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-23T21:54:38,294 DEBUG [RS:0;adda16083778:45163 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-23T21:54:38,296 INFO [master/adda16083778:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2024-11-23T21:54:38,301 DEBUG [master/adda16083778:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: adda16083778,36037,1732398876186 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2024-11-23T21:54:38,326 DEBUG [RS:0;adda16083778:45163 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-23T21:54:38,326 DEBUG [RS:0;adda16083778:45163 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-23T21:54:38,327 DEBUG [master/adda16083778:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/adda16083778:0, corePoolSize=5, maxPoolSize=5 2024-11-23T21:54:38,327 DEBUG [RS:1;adda16083778:42343 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-23T21:54:38,327 DEBUG [RS:1;adda16083778:42343 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-23T21:54:38,327 DEBUG [master/adda16083778:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/adda16083778:0, corePoolSize=5, maxPoolSize=5 2024-11-23T21:54:38,327 DEBUG [master/adda16083778:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/adda16083778:0, corePoolSize=5, maxPoolSize=5 2024-11-23T21:54:38,327 DEBUG [master/adda16083778:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/adda16083778:0, corePoolSize=5, maxPoolSize=5 2024-11-23T21:54:38,328 DEBUG [master/adda16083778:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/adda16083778:0, corePoolSize=10, maxPoolSize=10 2024-11-23T21:54:38,328 DEBUG [master/adda16083778:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/adda16083778:0, corePoolSize=1, maxPoolSize=1 2024-11-23T21:54:38,328 DEBUG [master/adda16083778:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/adda16083778:0, corePoolSize=2, maxPoolSize=2 2024-11-23T21:54:38,328 DEBUG [master/adda16083778:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/adda16083778:0, corePoolSize=1, maxPoolSize=1 2024-11-23T21:54:38,329 INFO [master/adda16083778:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732398908329 2024-11-23T21:54:38,331 INFO [master/adda16083778:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2024-11-23T21:54:38,332 INFO [master/adda16083778:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2024-11-23T21:54:38,334 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-23T21:54:38,335 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region 2024-11-23T21:54:38,335 INFO [master/adda16083778:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2024-11-23T21:54:38,336 INFO [master/adda16083778:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2024-11-23T21:54:38,336 INFO [master/adda16083778:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2024-11-23T21:54:38,336 INFO [master/adda16083778:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2024-11-23T21:54:38,337 INFO [master/adda16083778:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-23T21:54:38,338 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T21:54:38,339 INFO [master/adda16083778:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2024-11-23T21:54:38,339 INFO [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-11-23T21:54:38,340 INFO [master/adda16083778:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2024-11-23T21:54:38,341 INFO [master/adda16083778:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2024-11-23T21:54:38,342 INFO [master/adda16083778:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2024-11-23T21:54:38,343 INFO [master/adda16083778:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2024-11-23T21:54:38,345 DEBUG [master/adda16083778:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/adda16083778:0:becomeActiveMaster-HFileCleaner.large.0-1732398878344,5,FailOnTimeoutGroup] 2024-11-23T21:54:38,345 DEBUG [master/adda16083778:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/adda16083778:0:becomeActiveMaster-HFileCleaner.small.0-1732398878345,5,FailOnTimeoutGroup] 2024-11-23T21:54:38,346 INFO [master/adda16083778:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-23T21:54:38,346 INFO [master/adda16083778:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2024-11-23T21:54:38,347 INFO [master/adda16083778:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2024-11-23T21:54:38,347 INFO [master/adda16083778:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2024-11-23T21:54:38,349 DEBUG [RS:0;adda16083778:45163 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-23T21:54:38,349 DEBUG [RS:1;adda16083778:42343 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-23T21:54:38,350 DEBUG [RS:1;adda16083778:42343 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2f4d5703, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=adda16083778/172.17.0.2:0 2024-11-23T21:54:38,350 DEBUG [RS:0;adda16083778:45163 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@80e9a51, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=adda16083778/172.17.0.2:0 2024-11-23T21:54:38,359 INFO [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/93be71cd-4dcf-2478-daf8-a2306a879141/e0862929-e242-795d-1e0b-fe7766981a5b/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321 2024-11-23T21:54:38,360 INFO [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/93be71cd-4dcf-2478-daf8-a2306a879141/e0862929-e242-795d-1e0b-fe7766981a5b 2024-11-23T21:54:38,371 DEBUG [RS:1;adda16083778:42343 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:1;adda16083778:42343 2024-11-23T21:54:38,372 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-23T21:54:38,374 DEBUG [RS:0;adda16083778:45163 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;adda16083778:45163 2024-11-23T21:54:38,375 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family info of region 1588230740 2024-11-23T21:54:38,376 INFO [RS:1;adda16083778:42343 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-23T21:54:38,376 INFO [RS:0;adda16083778:45163 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-23T21:54:38,376 INFO [RS:1;adda16083778:42343 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-23T21:54:38,376 INFO [RS:0;adda16083778:45163 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-23T21:54:38,376 DEBUG [RS:1;adda16083778:42343 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-23T21:54:38,376 DEBUG [RS:0;adda16083778:45163 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-23T21:54:38,379 INFO [RS:1;adda16083778:42343 {}] regionserver.HRegionServer(2659): reportForDuty to master=adda16083778,36037,1732398876186 with port=42343, startcode=1732398877237 2024-11-23T21:54:38,379 INFO [RS:0;adda16083778:45163 {}] regionserver.HRegionServer(2659): reportForDuty to master=adda16083778,36037,1732398876186 with port=45163, startcode=1732398877077 2024-11-23T21:54:38,379 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-23T21:54:38,380 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T21:54:38,380 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-23T21:54:38,380 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family ns of region 1588230740 2024-11-23T21:54:38,385 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-23T21:54:38,385 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T21:54:38,385 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-23T21:54:38,386 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family rep_barrier of region 1588230740 2024-11-23T21:54:38,390 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-23T21:54:38,390 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T21:54:38,391 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-23T21:54:38,391 DEBUG [RS:0;adda16083778:45163 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-23T21:54:38,391 DEBUG [RS:1;adda16083778:42343 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-23T21:54:38,391 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family table of region 1588230740 2024-11-23T21:54:38,395 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-23T21:54:38,395 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T21:54:38,395 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-23T21:54:38,396 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-23T21:54:38,397 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/93be71cd-4dcf-2478-daf8-a2306a879141/e0862929-e242-795d-1e0b-fe7766981a5b/data/hbase/meta/1588230740 2024-11-23T21:54:38,397 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/93be71cd-4dcf-2478-daf8-a2306a879141/e0862929-e242-795d-1e0b-fe7766981a5b/data/hbase/meta/1588230740 2024-11-23T21:54:38,398 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-23T21:54:38,398 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-23T21:54:38,398 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-23T21:54:38,399 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-23T21:54:38,408 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/93be71cd-4dcf-2478-daf8-a2306a879141/e0862929-e242-795d-1e0b-fe7766981a5b/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-23T21:54:38,408 INFO [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=66150070, jitterRate=-0.01428714394569397}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-23T21:54:38,414 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740: Writing region info on filesystem at 1732398878372Initializing all the Stores at 1732398878372Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732398878372Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732398878375 (+3 ms)Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732398878375Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732398878375Cleaning up temporary data from old regions at 1732398878398 (+23 ms)Region opened successfully at 1732398878414 (+16 ms) 2024-11-23T21:54:38,415 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-23T21:54:38,415 INFO [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-23T21:54:38,415 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-23T21:54:38,415 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-23T21:54:38,415 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-23T21:54:38,417 INFO [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-23T21:54:38,417 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1732398878415Disabling compacts and flushes for region at 1732398878415Disabling writes for close at 1732398878415Writing region close event to WAL at 1732398878417 (+2 ms)Closed at 1732398878417 2024-11-23T21:54:38,420 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-23T21:54:38,420 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta 2024-11-23T21:54:38,428 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-11-23T21:54:38,437 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-23T21:54:38,439 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2024-11-23T21:54:38,469 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38171, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=RegionServerStatusService 2024-11-23T21:54:38,471 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39681, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=RegionServerStatusService 2024-11-23T21:54:38,482 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=36037 {}] master.ServerManager(363): Checking decommissioned status of RegionServer adda16083778,45163,1732398877077 2024-11-23T21:54:38,485 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=36037 {}] master.ServerManager(517): Registering regionserver=adda16083778,45163,1732398877077 2024-11-23T21:54:38,498 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=36037 {}] master.ServerManager(363): Checking decommissioned status of RegionServer adda16083778,42343,1732398877237 2024-11-23T21:54:38,498 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=36037 {}] master.ServerManager(517): Registering regionserver=adda16083778,42343,1732398877237 2024-11-23T21:54:38,502 DEBUG [RS:0;adda16083778:45163 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/93be71cd-4dcf-2478-daf8-a2306a879141/e0862929-e242-795d-1e0b-fe7766981a5b 2024-11-23T21:54:38,502 DEBUG [RS:1;adda16083778:42343 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/93be71cd-4dcf-2478-daf8-a2306a879141/e0862929-e242-795d-1e0b-fe7766981a5b 2024-11-23T21:54:38,502 DEBUG [RS:0;adda16083778:45163 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=file:/ 2024-11-23T21:54:38,502 DEBUG [RS:1;adda16083778:42343 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=file:/ 2024-11-23T21:54:38,502 DEBUG [RS:1;adda16083778:42343 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-23T21:54:38,502 DEBUG [RS:0;adda16083778:45163 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-23T21:54:38,574 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36037-0x1016afa97d20000, quorum=127.0.0.1:60787, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-23T21:54:38,574 DEBUG [RS:1;adda16083778:42343 {}] zookeeper.ZKUtil(111): regionserver:42343-0x1016afa97d20002, quorum=127.0.0.1:60787, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/adda16083778,42343,1732398877237 2024-11-23T21:54:38,574 DEBUG [RS:0;adda16083778:45163 {}] zookeeper.ZKUtil(111): regionserver:45163-0x1016afa97d20001, quorum=127.0.0.1:60787, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/adda16083778,45163,1732398877077 2024-11-23T21:54:38,575 WARN [RS:1;adda16083778:42343 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-23T21:54:38,575 WARN [RS:0;adda16083778:45163 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-23T21:54:38,575 INFO [RS:1;adda16083778:42343 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-23T21:54:38,575 INFO [RS:0;adda16083778:45163 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-23T21:54:38,575 DEBUG [RS:1;adda16083778:42343 {}] regionserver.HRegionServer(1793): logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/93be71cd-4dcf-2478-daf8-a2306a879141/e0862929-e242-795d-1e0b-fe7766981a5b/WALs/adda16083778,42343,1732398877237 2024-11-23T21:54:38,575 DEBUG [RS:0;adda16083778:45163 {}] regionserver.HRegionServer(1793): logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/93be71cd-4dcf-2478-daf8-a2306a879141/e0862929-e242-795d-1e0b-fe7766981a5b/WALs/adda16083778,45163,1732398877077 2024-11-23T21:54:38,579 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [adda16083778,45163,1732398877077] 2024-11-23T21:54:38,579 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [adda16083778,42343,1732398877237] 2024-11-23T21:54:38,590 WARN [adda16083778:36037 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-23T21:54:38,608 INFO [RS:1;adda16083778:42343 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-23T21:54:38,608 INFO [RS:0;adda16083778:45163 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-23T21:54:38,634 INFO [RS:0;adda16083778:45163 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-23T21:54:38,634 INFO [RS:1;adda16083778:42343 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-23T21:54:38,640 INFO [RS:0;adda16083778:45163 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-23T21:54:38,640 INFO [RS:1;adda16083778:42343 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-23T21:54:38,640 INFO [RS:0;adda16083778:45163 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-23T21:54:38,640 INFO [RS:1;adda16083778:42343 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-23T21:54:38,642 INFO [RS:0;adda16083778:45163 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-23T21:54:38,642 INFO [RS:1;adda16083778:42343 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-23T21:54:38,648 INFO [RS:1;adda16083778:42343 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-23T21:54:38,648 INFO [RS:0;adda16083778:45163 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-23T21:54:38,650 INFO [RS:0;adda16083778:45163 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-23T21:54:38,650 INFO [RS:1;adda16083778:42343 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-23T21:54:38,651 DEBUG [RS:0;adda16083778:45163 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/adda16083778:0, corePoolSize=1, maxPoolSize=1 2024-11-23T21:54:38,651 DEBUG [RS:1;adda16083778:42343 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/adda16083778:0, corePoolSize=1, maxPoolSize=1 2024-11-23T21:54:38,651 DEBUG [RS:0;adda16083778:45163 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/adda16083778:0, corePoolSize=1, maxPoolSize=1 2024-11-23T21:54:38,651 DEBUG [RS:1;adda16083778:42343 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/adda16083778:0, corePoolSize=1, maxPoolSize=1 2024-11-23T21:54:38,651 DEBUG [RS:0;adda16083778:45163 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/adda16083778:0, corePoolSize=1, maxPoolSize=1 2024-11-23T21:54:38,651 DEBUG [RS:1;adda16083778:42343 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/adda16083778:0, corePoolSize=1, maxPoolSize=1 2024-11-23T21:54:38,651 DEBUG [RS:0;adda16083778:45163 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/adda16083778:0, corePoolSize=1, maxPoolSize=1 2024-11-23T21:54:38,651 DEBUG [RS:1;adda16083778:42343 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/adda16083778:0, corePoolSize=1, maxPoolSize=1 2024-11-23T21:54:38,652 DEBUG [RS:0;adda16083778:45163 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/adda16083778:0, corePoolSize=1, maxPoolSize=1 2024-11-23T21:54:38,652 DEBUG [RS:1;adda16083778:42343 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/adda16083778:0, corePoolSize=1, maxPoolSize=1 2024-11-23T21:54:38,652 DEBUG [RS:0;adda16083778:45163 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/adda16083778:0, corePoolSize=2, maxPoolSize=2 2024-11-23T21:54:38,652 DEBUG [RS:1;adda16083778:42343 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/adda16083778:0, corePoolSize=2, maxPoolSize=2 2024-11-23T21:54:38,652 DEBUG [RS:0;adda16083778:45163 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/adda16083778:0, corePoolSize=1, maxPoolSize=1 2024-11-23T21:54:38,652 DEBUG [RS:1;adda16083778:42343 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/adda16083778:0, corePoolSize=1, maxPoolSize=1 2024-11-23T21:54:38,652 DEBUG [RS:0;adda16083778:45163 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/adda16083778:0, corePoolSize=1, maxPoolSize=1 2024-11-23T21:54:38,652 DEBUG [RS:1;adda16083778:42343 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/adda16083778:0, corePoolSize=1, maxPoolSize=1 2024-11-23T21:54:38,652 DEBUG [RS:0;adda16083778:45163 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/adda16083778:0, corePoolSize=1, maxPoolSize=1 2024-11-23T21:54:38,652 DEBUG [RS:1;adda16083778:42343 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/adda16083778:0, corePoolSize=1, maxPoolSize=1 2024-11-23T21:54:38,652 DEBUG [RS:0;adda16083778:45163 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/adda16083778:0, corePoolSize=1, maxPoolSize=1 2024-11-23T21:54:38,653 DEBUG [RS:1;adda16083778:42343 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/adda16083778:0, corePoolSize=1, maxPoolSize=1 2024-11-23T21:54:38,653 DEBUG [RS:0;adda16083778:45163 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/adda16083778:0, corePoolSize=1, maxPoolSize=1 2024-11-23T21:54:38,653 DEBUG [RS:1;adda16083778:42343 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/adda16083778:0, corePoolSize=1, maxPoolSize=1 2024-11-23T21:54:38,653 DEBUG [RS:0;adda16083778:45163 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/adda16083778:0, corePoolSize=1, maxPoolSize=1 2024-11-23T21:54:38,653 DEBUG [RS:1;adda16083778:42343 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/adda16083778:0, corePoolSize=1, maxPoolSize=1 2024-11-23T21:54:38,653 DEBUG [RS:0;adda16083778:45163 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/adda16083778:0, corePoolSize=3, maxPoolSize=3 2024-11-23T21:54:38,653 DEBUG [RS:1;adda16083778:42343 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/adda16083778:0, corePoolSize=3, maxPoolSize=3 2024-11-23T21:54:38,653 DEBUG [RS:0;adda16083778:45163 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/adda16083778:0, corePoolSize=3, maxPoolSize=3 2024-11-23T21:54:38,653 DEBUG [RS:1;adda16083778:42343 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/adda16083778:0, corePoolSize=3, maxPoolSize=3 2024-11-23T21:54:38,655 INFO [RS:0;adda16083778:45163 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-23T21:54:38,655 INFO [RS:0;adda16083778:45163 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-23T21:54:38,655 INFO [RS:0;adda16083778:45163 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-23T21:54:38,655 INFO [RS:0;adda16083778:45163 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-23T21:54:38,656 INFO [RS:0;adda16083778:45163 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-23T21:54:38,656 INFO [RS:0;adda16083778:45163 {}] hbase.ChoreService(168): Chore ScheduledChore name=adda16083778,45163,1732398877077-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-23T21:54:38,658 INFO [RS:1;adda16083778:42343 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-23T21:54:38,658 INFO [RS:1;adda16083778:42343 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-23T21:54:38,658 INFO [RS:1;adda16083778:42343 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-23T21:54:38,658 INFO [RS:1;adda16083778:42343 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-23T21:54:38,658 INFO [RS:1;adda16083778:42343 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-23T21:54:38,659 INFO [RS:1;adda16083778:42343 {}] hbase.ChoreService(168): Chore ScheduledChore name=adda16083778,42343,1732398877237-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-23T21:54:38,687 INFO [RS:0;adda16083778:45163 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-23T21:54:38,687 INFO [RS:1;adda16083778:42343 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-23T21:54:38,689 INFO [RS:1;adda16083778:42343 {}] hbase.ChoreService(168): Chore ScheduledChore name=adda16083778,42343,1732398877237-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-23T21:54:38,689 INFO [RS:0;adda16083778:45163 {}] hbase.ChoreService(168): Chore ScheduledChore name=adda16083778,45163,1732398877077-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-23T21:54:38,690 INFO [RS:1;adda16083778:42343 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-23T21:54:38,690 INFO [RS:0;adda16083778:45163 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-23T21:54:38,690 INFO [RS:0;adda16083778:45163 {}] regionserver.Replication(171): adda16083778,45163,1732398877077 started 2024-11-23T21:54:38,690 INFO [RS:1;adda16083778:42343 {}] regionserver.Replication(171): adda16083778,42343,1732398877237 started 2024-11-23T21:54:38,715 INFO [RS:0;adda16083778:45163 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-23T21:54:38,715 INFO [RS:1;adda16083778:42343 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-23T21:54:38,715 INFO [RS:0;adda16083778:45163 {}] regionserver.HRegionServer(1482): Serving as adda16083778,45163,1732398877077, RpcServer on adda16083778/172.17.0.2:45163, sessionid=0x1016afa97d20001 2024-11-23T21:54:38,715 INFO [RS:1;adda16083778:42343 {}] regionserver.HRegionServer(1482): Serving as adda16083778,42343,1732398877237, RpcServer on adda16083778/172.17.0.2:42343, sessionid=0x1016afa97d20002 2024-11-23T21:54:38,716 DEBUG [RS:0;adda16083778:45163 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-23T21:54:38,716 DEBUG [RS:1;adda16083778:42343 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-23T21:54:38,716 DEBUG [RS:0;adda16083778:45163 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager adda16083778,45163,1732398877077 2024-11-23T21:54:38,716 DEBUG [RS:1;adda16083778:42343 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager adda16083778,42343,1732398877237 2024-11-23T21:54:38,717 DEBUG [RS:1;adda16083778:42343 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'adda16083778,42343,1732398877237' 2024-11-23T21:54:38,717 DEBUG [RS:0;adda16083778:45163 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'adda16083778,45163,1732398877077' 2024-11-23T21:54:38,717 DEBUG [RS:1;adda16083778:42343 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-23T21:54:38,717 DEBUG [RS:0;adda16083778:45163 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-23T21:54:38,718 DEBUG [RS:0;adda16083778:45163 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-23T21:54:38,718 DEBUG [RS:1;adda16083778:42343 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-23T21:54:38,719 DEBUG [RS:1;adda16083778:42343 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-23T21:54:38,719 DEBUG [RS:1;adda16083778:42343 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-23T21:54:38,719 DEBUG [RS:0;adda16083778:45163 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-23T21:54:38,719 DEBUG [RS:0;adda16083778:45163 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-23T21:54:38,719 DEBUG [RS:0;adda16083778:45163 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager adda16083778,45163,1732398877077 2024-11-23T21:54:38,719 DEBUG [RS:1;adda16083778:42343 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager adda16083778,42343,1732398877237 2024-11-23T21:54:38,719 DEBUG [RS:0;adda16083778:45163 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'adda16083778,45163,1732398877077' 2024-11-23T21:54:38,719 DEBUG [RS:1;adda16083778:42343 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'adda16083778,42343,1732398877237' 2024-11-23T21:54:38,719 DEBUG [RS:0;adda16083778:45163 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-23T21:54:38,719 DEBUG [RS:1;adda16083778:42343 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-23T21:54:38,720 DEBUG [RS:0;adda16083778:45163 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-23T21:54:38,720 DEBUG [RS:1;adda16083778:42343 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-23T21:54:38,721 DEBUG [RS:0;adda16083778:45163 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-23T21:54:38,721 DEBUG [RS:1;adda16083778:42343 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-23T21:54:38,721 INFO [RS:0;adda16083778:45163 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-23T21:54:38,721 INFO [RS:1;adda16083778:42343 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-23T21:54:38,721 INFO [RS:0;adda16083778:45163 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-23T21:54:38,721 INFO [RS:1;adda16083778:42343 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-23T21:54:38,827 INFO [RS:1;adda16083778:42343 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-23T21:54:38,827 INFO [RS:0;adda16083778:45163 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-23T21:54:38,828 INFO [RS:1;adda16083778:42343 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=adda16083778%2C42343%2C1732398877237, suffix=, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/93be71cd-4dcf-2478-daf8-a2306a879141/e0862929-e242-795d-1e0b-fe7766981a5b/WALs/adda16083778,42343,1732398877237, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/93be71cd-4dcf-2478-daf8-a2306a879141/e0862929-e242-795d-1e0b-fe7766981a5b/oldWALs, maxLogs=55 2024-11-23T21:54:38,828 INFO [RS:0;adda16083778:45163 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=adda16083778%2C45163%2C1732398877077, suffix=, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/93be71cd-4dcf-2478-daf8-a2306a879141/e0862929-e242-795d-1e0b-fe7766981a5b/WALs/adda16083778,45163,1732398877077, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/93be71cd-4dcf-2478-daf8-a2306a879141/e0862929-e242-795d-1e0b-fe7766981a5b/oldWALs, maxLogs=55 2024-11-23T21:54:38,849 INFO [RS:0;adda16083778:45163 {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/93be71cd-4dcf-2478-daf8-a2306a879141/e0862929-e242-795d-1e0b-fe7766981a5b/WALs/adda16083778,45163,1732398877077/adda16083778%2C45163%2C1732398877077.1732398878832 2024-11-23T21:54:38,849 INFO [RS:1;adda16083778:42343 {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/93be71cd-4dcf-2478-daf8-a2306a879141/e0862929-e242-795d-1e0b-fe7766981a5b/WALs/adda16083778,42343,1732398877237/adda16083778%2C42343%2C1732398877237.1732398878832 2024-11-23T21:54:38,849 DEBUG [RS:1;adda16083778:42343 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-11-23T21:54:38,849 DEBUG [RS:0;adda16083778:45163 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-11-23T21:54:39,093 DEBUG [adda16083778:36037 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=2, allServersCount=2 2024-11-23T21:54:39,106 DEBUG [adda16083778:36037 {}] balancer.BalancerClusterState(204): Hosts are {adda16083778=0} racks are {/default-rack=0} 2024-11-23T21:54:39,109 DEBUG [adda16083778:36037 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-11-23T21:54:39,109 DEBUG [adda16083778:36037 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-11-23T21:54:39,109 DEBUG [adda16083778:36037 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-11-23T21:54:39,109 DEBUG [adda16083778:36037 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-11-23T21:54:39,109 INFO [adda16083778:36037 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-11-23T21:54:39,109 INFO [adda16083778:36037 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-11-23T21:54:39,109 DEBUG [adda16083778:36037 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-11-23T21:54:39,115 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=adda16083778,45163,1732398877077 2024-11-23T21:54:39,120 INFO [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as adda16083778,45163,1732398877077, state=OPENING 2024-11-23T21:54:39,174 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2024-11-23T21:54:39,182 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45163-0x1016afa97d20001, quorum=127.0.0.1:60787, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-23T21:54:39,182 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42343-0x1016afa97d20002, quorum=127.0.0.1:60787, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-23T21:54:39,182 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36037-0x1016afa97d20000, quorum=127.0.0.1:60787, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-23T21:54:39,184 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-23T21:54:39,184 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-23T21:54:39,184 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-23T21:54:39,186 DEBUG [PEWorker-3 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-23T21:54:39,188 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=adda16083778,45163,1732398877077}] 2024-11-23T21:54:39,356 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-23T21:54:39,359 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43665, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-23T21:54:39,366 INFO [RS_OPEN_META-regionserver/adda16083778:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-11-23T21:54:39,367 INFO [RS_OPEN_META-regionserver/adda16083778:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-23T21:54:39,367 INFO [RS_OPEN_META-regionserver/adda16083778:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-11-23T21:54:39,368 INFO [RS_OPEN_META-regionserver/adda16083778:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=adda16083778%2C45163%2C1732398877077.meta, suffix=.meta, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/93be71cd-4dcf-2478-daf8-a2306a879141/e0862929-e242-795d-1e0b-fe7766981a5b/WALs/adda16083778,45163,1732398877077, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/93be71cd-4dcf-2478-daf8-a2306a879141/e0862929-e242-795d-1e0b-fe7766981a5b/oldWALs, maxLogs=55 2024-11-23T21:54:39,387 INFO [RS_OPEN_META-regionserver/adda16083778:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/93be71cd-4dcf-2478-daf8-a2306a879141/e0862929-e242-795d-1e0b-fe7766981a5b/WALs/adda16083778,45163,1732398877077/adda16083778%2C45163%2C1732398877077.meta.1732398879370.meta 2024-11-23T21:54:39,388 DEBUG [RS_OPEN_META-regionserver/adda16083778:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-11-23T21:54:39,388 DEBUG [RS_OPEN_META-regionserver/adda16083778:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-11-23T21:54:39,390 DEBUG [RS_OPEN_META-regionserver/adda16083778:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-11-23T21:54:39,392 DEBUG [RS_OPEN_META-regionserver/adda16083778:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-11-23T21:54:39,396 INFO [RS_OPEN_META-regionserver/adda16083778:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-11-23T21:54:39,400 DEBUG [RS_OPEN_META-regionserver/adda16083778:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-11-23T21:54:39,400 DEBUG [RS_OPEN_META-regionserver/adda16083778:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-23T21:54:39,400 DEBUG [RS_OPEN_META-regionserver/adda16083778:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-11-23T21:54:39,401 DEBUG [RS_OPEN_META-regionserver/adda16083778:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-11-23T21:54:39,402 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family info of region 1588230740 2024-11-23T21:54:39,403 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-23T21:54:39,403 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T21:54:39,403 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-23T21:54:39,404 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family ns of region 1588230740 2024-11-23T21:54:39,404 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-23T21:54:39,404 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T21:54:39,404 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-23T21:54:39,405 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family rep_barrier of region 1588230740 2024-11-23T21:54:39,405 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-23T21:54:39,405 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T21:54:39,405 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-23T21:54:39,406 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family table of region 1588230740 2024-11-23T21:54:39,406 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-23T21:54:39,406 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T21:54:39,407 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-23T21:54:39,407 DEBUG [RS_OPEN_META-regionserver/adda16083778:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-23T21:54:39,408 DEBUG [RS_OPEN_META-regionserver/adda16083778:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/93be71cd-4dcf-2478-daf8-a2306a879141/e0862929-e242-795d-1e0b-fe7766981a5b/data/hbase/meta/1588230740 2024-11-23T21:54:39,409 DEBUG [RS_OPEN_META-regionserver/adda16083778:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/93be71cd-4dcf-2478-daf8-a2306a879141/e0862929-e242-795d-1e0b-fe7766981a5b/data/hbase/meta/1588230740 2024-11-23T21:54:39,409 DEBUG [RS_OPEN_META-regionserver/adda16083778:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-23T21:54:39,409 DEBUG [RS_OPEN_META-regionserver/adda16083778:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-23T21:54:39,410 DEBUG [RS_OPEN_META-regionserver/adda16083778:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-23T21:54:39,411 DEBUG [RS_OPEN_META-regionserver/adda16083778:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-23T21:54:39,412 INFO [RS_OPEN_META-regionserver/adda16083778:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=72421438, jitterRate=0.07916352152824402}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-23T21:54:39,412 DEBUG [RS_OPEN_META-regionserver/adda16083778:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-11-23T21:54:39,413 DEBUG [RS_OPEN_META-regionserver/adda16083778:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1732398879401Writing region info on filesystem at 1732398879401Initializing all the Stores at 1732398879402 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732398879402Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732398879402Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732398879402Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732398879402Cleaning up temporary data from old regions at 1732398879409 (+7 ms)Running coprocessor post-open hooks at 1732398879412 (+3 ms)Region opened successfully at 1732398879413 (+1 ms) 2024-11-23T21:54:39,419 INFO [RS_OPEN_META-regionserver/adda16083778:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1732398879349 2024-11-23T21:54:39,429 DEBUG [RS_OPEN_META-regionserver/adda16083778:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-11-23T21:54:39,430 INFO [RS_OPEN_META-regionserver/adda16083778:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-11-23T21:54:39,430 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=adda16083778,45163,1732398877077 2024-11-23T21:54:39,432 INFO [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as adda16083778,45163,1732398877077, state=OPEN 2024-11-23T21:54:39,491 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45163-0x1016afa97d20001, quorum=127.0.0.1:60787, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-23T21:54:39,491 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36037-0x1016afa97d20000, quorum=127.0.0.1:60787, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-23T21:54:39,491 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42343-0x1016afa97d20002, quorum=127.0.0.1:60787, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-23T21:54:39,491 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-23T21:54:39,491 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-23T21:54:39,491 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-23T21:54:39,492 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=adda16083778,45163,1732398877077 2024-11-23T21:54:39,497 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2 2024-11-23T21:54:39,498 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=adda16083778,45163,1732398877077 in 305 msec 2024-11-23T21:54:39,503 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1 2024-11-23T21:54:39,503 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 1.0720 sec 2024-11-23T21:54:39,504 DEBUG [PEWorker-2 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-23T21:54:39,504 INFO [PEWorker-2 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces 2024-11-23T21:54:39,522 DEBUG [PEWorker-2 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-23T21:54:39,523 DEBUG [PEWorker-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=adda16083778,45163,1732398877077, seqNum=-1] 2024-11-23T21:54:39,540 DEBUG [PEWorker-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-23T21:54:39,542 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:32801, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-23T21:54:39,557 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 1.3200 sec 2024-11-23T21:54:39,557 INFO [master/adda16083778:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1732398879557, completionTime=-1 2024-11-23T21:54:39,560 INFO [master/adda16083778:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=2; waited=0ms, expected min=2 server(s), max=2 server(s), master is running 2024-11-23T21:54:39,560 DEBUG [master/adda16083778:0:becomeActiveMaster {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-23T21:54:39,587 INFO [master/adda16083778:0:becomeActiveMaster {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-11-23T21:54:39,587 INFO [master/adda16083778:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732398939587 2024-11-23T21:54:39,588 INFO [master/adda16083778:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732398999587 2024-11-23T21:54:39,588 INFO [master/adda16083778:0:becomeActiveMaster {}] assignment.AssignmentManager(1783): Joined the cluster in 27 msec 2024-11-23T21:54:39,590 DEBUG [master/adda16083778:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 1588230740 changed from -1.0 to 0.0, refreshing cache 2024-11-23T21:54:39,598 INFO [master/adda16083778:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=adda16083778,36037,1732398876186-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-23T21:54:39,598 INFO [master/adda16083778:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=adda16083778,36037,1732398876186-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-23T21:54:39,598 INFO [master/adda16083778:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=adda16083778,36037,1732398876186-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-23T21:54:39,600 INFO [master/adda16083778:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-adda16083778:36037, period=300000, unit=MILLISECONDS is enabled. 2024-11-23T21:54:39,601 INFO [master/adda16083778:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2024-11-23T21:54:39,601 INFO [master/adda16083778:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled. 2024-11-23T21:54:39,606 DEBUG [master/adda16083778:0.Chore.1 {}] janitor.CatalogJanitor(180): 2024-11-23T21:54:39,631 INFO [master/adda16083778:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 2.262sec 2024-11-23T21:54:39,632 INFO [master/adda16083778:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2024-11-23T21:54:39,634 INFO [master/adda16083778:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2024-11-23T21:54:39,635 INFO [master/adda16083778:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2024-11-23T21:54:39,636 INFO [master/adda16083778:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2024-11-23T21:54:39,636 INFO [master/adda16083778:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2024-11-23T21:54:39,637 INFO [master/adda16083778:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=adda16083778,36037,1732398876186-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-23T21:54:39,638 INFO [master/adda16083778:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=adda16083778,36037,1732398876186-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2024-11-23T21:54:39,639 DEBUG [master/adda16083778:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds 2024-11-23T21:54:39,640 INFO [master/adda16083778:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2024-11-23T21:54:39,641 INFO [master/adda16083778:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=adda16083778,36037,1732398876186-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-23T21:54:39,702 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@319823c7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T21:54:39,705 DEBUG [Time-limited test {}] nio.NioEventLoop(110): -Dio.netty.noKeySetOptimization: false 2024-11-23T21:54:39,705 DEBUG [Time-limited test {}] nio.NioEventLoop(111): -Dio.netty.selectorAutoRebuildThreshold: 512 2024-11-23T21:54:39,709 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request adda16083778,36037,-1 for getting cluster id 2024-11-23T21:54:39,712 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-23T21:54:39,720 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '3cb83c07-7b6f-42b5-a954-50dac1670ae4' 2024-11-23T21:54:39,722 DEBUG [RPCClient-NioEventLoopGroup-5-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-23T21:54:39,723 DEBUG [RPCClient-NioEventLoopGroup-5-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "3cb83c07-7b6f-42b5-a954-50dac1670ae4" 2024-11-23T21:54:39,724 DEBUG [RPCClient-NioEventLoopGroup-5-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6011b7b9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T21:54:39,725 DEBUG [RPCClient-NioEventLoopGroup-5-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [adda16083778,36037,-1] 2024-11-23T21:54:39,727 DEBUG [RPCClient-NioEventLoopGroup-5-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-23T21:54:39,729 DEBUG [RPCClient-NioEventLoopGroup-5-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T21:54:39,730 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43754, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-23T21:54:39,733 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@19e12cb1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T21:54:39,733 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-23T21:54:39,740 DEBUG [RPCClient-NioEventLoopGroup-5-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=adda16083778,45163,1732398877077, seqNum=-1] 2024-11-23T21:54:39,741 DEBUG [RPCClient-NioEventLoopGroup-5-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-23T21:54:39,744 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46258, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-23T21:54:39,765 INFO [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=adda16083778,36037,1732398876186 2024-11-23T21:54:39,770 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-23T21:54:39,774 DEBUG [RPCClient-NioEventLoopGroup-5-2 {}] client.AsyncConnectionImpl(321): The fetched master address is adda16083778,36037,1732398876186 2024-11-23T21:54:39,777 DEBUG [RPCClient-NioEventLoopGroup-5-2 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@318d10c4 2024-11-23T21:54:39,778 DEBUG [RPCClient-NioEventLoopGroup-5-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-23T21:54:39,780 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43768, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-23T21:54:39,794 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36037 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'testBlockEvictionOnRegionMove1732398879766', {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-23T21:54:39,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36037 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=testBlockEvictionOnRegionMove1732398879766 2024-11-23T21:54:39,802 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=testBlockEvictionOnRegionMove1732398879766 execute state=CREATE_TABLE_PRE_OPERATION 2024-11-23T21:54:39,804 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36037 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "testBlockEvictionOnRegionMove1732398879766" procId is: 4 2024-11-23T21:54:39,804 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T21:54:39,806 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=testBlockEvictionOnRegionMove1732398879766 execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-23T21:54:39,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36037 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-23T21:54:39,821 INFO [RegionOpenAndInit-testBlockEvictionOnRegionMove1732398879766-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => da33864fc4ea86d45bff94a07435b68e, NAME => 'testBlockEvictionOnRegionMove1732398879766,,1732398879781.da33864fc4ea86d45bff94a07435b68e.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testBlockEvictionOnRegionMove1732398879766', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/93be71cd-4dcf-2478-daf8-a2306a879141/e0862929-e242-795d-1e0b-fe7766981a5b 2024-11-23T21:54:39,827 DEBUG [RegionOpenAndInit-testBlockEvictionOnRegionMove1732398879766-pool-0 {}] regionserver.HRegion(898): Instantiated testBlockEvictionOnRegionMove1732398879766,,1732398879781.da33864fc4ea86d45bff94a07435b68e.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-23T21:54:39,827 DEBUG [RegionOpenAndInit-testBlockEvictionOnRegionMove1732398879766-pool-0 {}] regionserver.HRegion(1722): Closing da33864fc4ea86d45bff94a07435b68e, disabling compactions & flushes 2024-11-23T21:54:39,827 INFO [RegionOpenAndInit-testBlockEvictionOnRegionMove1732398879766-pool-0 {}] regionserver.HRegion(1755): Closing region testBlockEvictionOnRegionMove1732398879766,,1732398879781.da33864fc4ea86d45bff94a07435b68e. 2024-11-23T21:54:39,827 DEBUG [RegionOpenAndInit-testBlockEvictionOnRegionMove1732398879766-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on testBlockEvictionOnRegionMove1732398879766,,1732398879781.da33864fc4ea86d45bff94a07435b68e. 2024-11-23T21:54:39,827 DEBUG [RegionOpenAndInit-testBlockEvictionOnRegionMove1732398879766-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on testBlockEvictionOnRegionMove1732398879766,,1732398879781.da33864fc4ea86d45bff94a07435b68e. after waiting 0 ms 2024-11-23T21:54:39,827 DEBUG [RegionOpenAndInit-testBlockEvictionOnRegionMove1732398879766-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region testBlockEvictionOnRegionMove1732398879766,,1732398879781.da33864fc4ea86d45bff94a07435b68e. 2024-11-23T21:54:39,827 INFO [RegionOpenAndInit-testBlockEvictionOnRegionMove1732398879766-pool-0 {}] regionserver.HRegion(1973): Closed testBlockEvictionOnRegionMove1732398879766,,1732398879781.da33864fc4ea86d45bff94a07435b68e. 2024-11-23T21:54:39,827 DEBUG [RegionOpenAndInit-testBlockEvictionOnRegionMove1732398879766-pool-0 {}] regionserver.HRegion(1676): Region close journal for da33864fc4ea86d45bff94a07435b68e: Waiting for close lock at 1732398879827Disabling compacts and flushes for region at 1732398879827Disabling writes for close at 1732398879827Writing region close event to WAL at 1732398879827Closed at 1732398879827 2024-11-23T21:54:39,829 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=testBlockEvictionOnRegionMove1732398879766 execute state=CREATE_TABLE_ADD_TO_META 2024-11-23T21:54:39,835 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testBlockEvictionOnRegionMove1732398879766,,1732398879781.da33864fc4ea86d45bff94a07435b68e.","families":{"info":[{"qualifier":"regioninfo","vlen":76,"tag":[],"timestamp":"1732398879829"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732398879829"}]},"ts":"1732398879829"} 2024-11-23T21:54:39,839 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-23T21:54:39,840 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=testBlockEvictionOnRegionMove1732398879766 execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-23T21:54:39,843 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testBlockEvictionOnRegionMove1732398879766","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732398879841"}]},"ts":"1732398879841"} 2024-11-23T21:54:39,848 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=testBlockEvictionOnRegionMove1732398879766, state=ENABLING in hbase:meta 2024-11-23T21:54:39,848 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(204): Hosts are {adda16083778=0} racks are {/default-rack=0} 2024-11-23T21:54:39,849 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-11-23T21:54:39,849 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-11-23T21:54:39,849 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-11-23T21:54:39,849 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-11-23T21:54:39,849 INFO [PEWorker-3 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-11-23T21:54:39,849 INFO [PEWorker-3 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-11-23T21:54:39,849 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-11-23T21:54:39,851 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1732398879766, region=da33864fc4ea86d45bff94a07435b68e, ASSIGN}] 2024-11-23T21:54:39,853 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1732398879766, region=da33864fc4ea86d45bff94a07435b68e, ASSIGN 2024-11-23T21:54:39,854 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1732398879766, region=da33864fc4ea86d45bff94a07435b68e, ASSIGN; state=OFFLINE, location=adda16083778,45163,1732398877077; forceNewPlan=false, retain=false 2024-11-23T21:54:39,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36037 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-23T21:54:40,011 INFO [adda16083778:36037 {}] balancer.BaseLoadBalancer(388): Reassigned 1 regions. 1 retained the pre-restart assignment. 2024-11-23T21:54:40,013 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=da33864fc4ea86d45bff94a07435b68e, regionState=OPENING, regionLocation=adda16083778,45163,1732398877077 2024-11-23T21:54:40,017 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-4-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1732398879766, region=da33864fc4ea86d45bff94a07435b68e, ASSIGN because future has completed 2024-11-23T21:54:40,018 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure da33864fc4ea86d45bff94a07435b68e, server=adda16083778,45163,1732398877077}] 2024-11-23T21:54:40,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36037 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-23T21:54:40,173 INFO [RS_OPEN_REGION-regionserver/adda16083778:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(132): Open testBlockEvictionOnRegionMove1732398879766,,1732398879781.da33864fc4ea86d45bff94a07435b68e. 2024-11-23T21:54:40,174 DEBUG [RS_OPEN_REGION-regionserver/adda16083778:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7752): Opening region: {ENCODED => da33864fc4ea86d45bff94a07435b68e, NAME => 'testBlockEvictionOnRegionMove1732398879766,,1732398879781.da33864fc4ea86d45bff94a07435b68e.', STARTKEY => '', ENDKEY => ''} 2024-11-23T21:54:40,174 DEBUG [RS_OPEN_REGION-regionserver/adda16083778:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testBlockEvictionOnRegionMove1732398879766 da33864fc4ea86d45bff94a07435b68e 2024-11-23T21:54:40,174 DEBUG [RS_OPEN_REGION-regionserver/adda16083778:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(898): Instantiated testBlockEvictionOnRegionMove1732398879766,,1732398879781.da33864fc4ea86d45bff94a07435b68e.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-23T21:54:40,174 DEBUG [RS_OPEN_REGION-regionserver/adda16083778:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7794): checking encryption for da33864fc4ea86d45bff94a07435b68e 2024-11-23T21:54:40,174 DEBUG [RS_OPEN_REGION-regionserver/adda16083778:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7797): checking classloading for da33864fc4ea86d45bff94a07435b68e 2024-11-23T21:54:40,175 INFO [StoreOpener-da33864fc4ea86d45bff94a07435b68e-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family family of region da33864fc4ea86d45bff94a07435b68e 2024-11-23T21:54:40,179 INFO [StoreOpener-da33864fc4ea86d45bff94a07435b68e-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region da33864fc4ea86d45bff94a07435b68e columnFamilyName family 2024-11-23T21:54:40,180 DEBUG [StoreOpener-da33864fc4ea86d45bff94a07435b68e-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T21:54:40,180 INFO [StoreOpener-da33864fc4ea86d45bff94a07435b68e-1 {}] regionserver.HStore(327): Store=da33864fc4ea86d45bff94a07435b68e/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-23T21:54:40,180 DEBUG [RS_OPEN_REGION-regionserver/adda16083778:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1038): replaying wal for da33864fc4ea86d45bff94a07435b68e 2024-11-23T21:54:40,182 DEBUG [RS_OPEN_REGION-regionserver/adda16083778:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/93be71cd-4dcf-2478-daf8-a2306a879141/e0862929-e242-795d-1e0b-fe7766981a5b/data/default/testBlockEvictionOnRegionMove1732398879766/da33864fc4ea86d45bff94a07435b68e 2024-11-23T21:54:40,182 DEBUG [RS_OPEN_REGION-regionserver/adda16083778:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/93be71cd-4dcf-2478-daf8-a2306a879141/e0862929-e242-795d-1e0b-fe7766981a5b/data/default/testBlockEvictionOnRegionMove1732398879766/da33864fc4ea86d45bff94a07435b68e 2024-11-23T21:54:40,182 DEBUG [RS_OPEN_REGION-regionserver/adda16083778:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1048): stopping wal replay for da33864fc4ea86d45bff94a07435b68e 2024-11-23T21:54:40,182 DEBUG [RS_OPEN_REGION-regionserver/adda16083778:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1060): Cleaning up temporary data for da33864fc4ea86d45bff94a07435b68e 2024-11-23T21:54:40,183 DEBUG [RS_OPEN_REGION-regionserver/adda16083778:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1093): writing seq id for da33864fc4ea86d45bff94a07435b68e 2024-11-23T21:54:40,188 DEBUG [RS_OPEN_REGION-regionserver/adda16083778:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/93be71cd-4dcf-2478-daf8-a2306a879141/e0862929-e242-795d-1e0b-fe7766981a5b/data/default/testBlockEvictionOnRegionMove1732398879766/da33864fc4ea86d45bff94a07435b68e/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-23T21:54:40,189 INFO [RS_OPEN_REGION-regionserver/adda16083778:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1114): Opened da33864fc4ea86d45bff94a07435b68e; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=66865174, jitterRate=-0.0036312639713287354}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-23T21:54:40,189 DEBUG [RS_OPEN_REGION-regionserver/adda16083778:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1122): Running coprocessor post-open hooks for da33864fc4ea86d45bff94a07435b68e 2024-11-23T21:54:40,190 DEBUG [RS_OPEN_REGION-regionserver/adda16083778:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1006): Region open journal for da33864fc4ea86d45bff94a07435b68e: Running coprocessor pre-open hook at 1732398880175Writing region info on filesystem at 1732398880175Initializing all the Stores at 1732398880175Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732398880175Cleaning up temporary data from old regions at 1732398880182 (+7 ms)Running coprocessor post-open hooks at 1732398880189 (+7 ms)Region opened successfully at 1732398880190 (+1 ms) 2024-11-23T21:54:40,191 INFO [RS_OPEN_REGION-regionserver/adda16083778:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2236): Post open deploy tasks for testBlockEvictionOnRegionMove1732398879766,,1732398879781.da33864fc4ea86d45bff94a07435b68e., pid=6, masterSystemTime=1732398880170 2024-11-23T21:54:40,194 DEBUG [RS_OPEN_REGION-regionserver/adda16083778:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2266): Finished post open deploy task for testBlockEvictionOnRegionMove1732398879766,,1732398879781.da33864fc4ea86d45bff94a07435b68e. 2024-11-23T21:54:40,194 INFO [RS_OPEN_REGION-regionserver/adda16083778:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(153): Opened testBlockEvictionOnRegionMove1732398879766,,1732398879781.da33864fc4ea86d45bff94a07435b68e. 2024-11-23T21:54:40,194 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=da33864fc4ea86d45bff94a07435b68e, regionState=OPEN, openSeqNum=2, regionLocation=adda16083778,45163,1732398877077 2024-11-23T21:54:40,198 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-4-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure da33864fc4ea86d45bff94a07435b68e, server=adda16083778,45163,1732398877077 because future has completed 2024-11-23T21:54:40,201 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=6, resume processing ppid=5 2024-11-23T21:54:40,202 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure da33864fc4ea86d45bff94a07435b68e, server=adda16083778,45163,1732398877077 in 181 msec 2024-11-23T21:54:40,203 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=5, resume processing ppid=4 2024-11-23T21:54:40,203 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1732398879766, region=da33864fc4ea86d45bff94a07435b68e, ASSIGN in 350 msec 2024-11-23T21:54:40,204 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=testBlockEvictionOnRegionMove1732398879766 execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-23T21:54:40,204 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testBlockEvictionOnRegionMove1732398879766","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732398880204"}]},"ts":"1732398880204"} 2024-11-23T21:54:40,207 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=testBlockEvictionOnRegionMove1732398879766, state=ENABLED in hbase:meta 2024-11-23T21:54:40,208 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=testBlockEvictionOnRegionMove1732398879766 execute state=CREATE_TABLE_POST_OPERATION 2024-11-23T21:54:40,210 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=testBlockEvictionOnRegionMove1732398879766 in 412 msec 2024-11-23T21:54:40,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36037 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-23T21:54:40,440 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table testBlockEvictionOnRegionMove1732398879766 get assigned. Timeout = 60000ms 2024-11-23T21:54:40,440 INFO [RPCClient-NioEventLoopGroup-5-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:testBlockEvictionOnRegionMove1732398879766 completed 2024-11-23T21:54:40,443 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-23T21:54:40,448 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table testBlockEvictionOnRegionMove1732398879766 assigned to meta. Checking AM states. 2024-11-23T21:54:40,449 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-23T21:54:40,449 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table testBlockEvictionOnRegionMove1732398879766 assigned. 2024-11-23T21:54:40,457 DEBUG [RPCClient-NioEventLoopGroup-5-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testBlockEvictionOnRegionMove1732398879766', row='row1', locateType=CURRENT is [region=testBlockEvictionOnRegionMove1732398879766,,1732398879781.da33864fc4ea86d45bff94a07435b68e., hostname=adda16083778,45163,1732398877077, seqNum=2] 2024-11-23T21:54:40,469 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing da33864fc4ea86d45bff94a07435b68e 1/1 column families, dataSize=78 B heapSize=480 B 2024-11-23T21:54:40,492 DEBUG [Time-limited test {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 89e74abd0154460d901380db484b4f07 2024-11-23T21:54:40,526 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/93be71cd-4dcf-2478-daf8-a2306a879141/e0862929-e242-795d-1e0b-fe7766981a5b/data/default/testBlockEvictionOnRegionMove1732398879766/da33864fc4ea86d45bff94a07435b68e/.tmp/family/89e74abd0154460d901380db484b4f07 is 43, key is row1/family:qf1/1/Put/seqid=0 2024-11-23T21:54:40,531 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=78 B at sequenceid=6 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/93be71cd-4dcf-2478-daf8-a2306a879141/e0862929-e242-795d-1e0b-fe7766981a5b/data/default/testBlockEvictionOnRegionMove1732398879766/da33864fc4ea86d45bff94a07435b68e/.tmp/family/89e74abd0154460d901380db484b4f07 2024-11-23T21:54:40,563 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/93be71cd-4dcf-2478-daf8-a2306a879141/e0862929-e242-795d-1e0b-fe7766981a5b/data/default/testBlockEvictionOnRegionMove1732398879766/da33864fc4ea86d45bff94a07435b68e/.tmp/family/89e74abd0154460d901380db484b4f07 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/93be71cd-4dcf-2478-daf8-a2306a879141/e0862929-e242-795d-1e0b-fe7766981a5b/data/default/testBlockEvictionOnRegionMove1732398879766/da33864fc4ea86d45bff94a07435b68e/family/89e74abd0154460d901380db484b4f07 2024-11-23T21:54:40,564 DEBUG [Time-limited test {}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/93be71cd-4dcf-2478-daf8-a2306a879141/e0862929-e242-795d-1e0b-fe7766981a5b/data/default/testBlockEvictionOnRegionMove1732398879766/da33864fc4ea86d45bff94a07435b68e/family/89e74abd0154460d901380db484b4f07, delay=922 ms 2024-11-23T21:54:40,566 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/93be71cd-4dcf-2478-daf8-a2306a879141/e0862929-e242-795d-1e0b-fe7766981a5b/data/default/testBlockEvictionOnRegionMove1732398879766/da33864fc4ea86d45bff94a07435b68e/family/89e74abd0154460d901380db484b4f07, entries=2, sequenceid=6, filesize=4.9 K 2024-11-23T21:54:40,572 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~78 B/78, heapSize ~464 B/464, currentSize=0 B/0 for da33864fc4ea86d45bff94a07435b68e in 103ms, sequenceid=6, compaction requested=false 2024-11-23T21:54:40,573 DEBUG [Time-limited test {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'testBlockEvictionOnRegionMove1732398879766' 2024-11-23T21:54:40,574 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for da33864fc4ea86d45bff94a07435b68e: 2024-11-23T21:54:41,491 DEBUG [hfile-prefetch-1732398880564 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 89e74abd0154460d901380db484b4f07_0. 2024-11-23T21:54:41,497 DEBUG [hfile-prefetch-1732398880564 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 89e74abd0154460d901380db484b4f07_117. 2024-11-23T21:54:41,498 DEBUG [hfile-prefetch-1732398880564 {}] bucket.BucketCache(2281): Notifying caching completed for file file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/93be71cd-4dcf-2478-daf8-a2306a879141/e0862929-e242-795d-1e0b-fe7766981a5b/data/default/testBlockEvictionOnRegionMove1732398879766/da33864fc4ea86d45bff94a07435b68e/family/89e74abd0154460d901380db484b4f07, with total blocks 2, and data blocks 1 2024-11-23T21:54:41,498 DEBUG [hfile-prefetch-1732398880564 {}] bucket.BucketCache(2285): iterating over 0 entries in the backing map 2024-11-23T21:54:41,498 DEBUG [hfile-prefetch-1732398880564 {}] bucket.BucketCache(2308): Prefetch executor completed for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/93be71cd-4dcf-2478-daf8-a2306a879141/e0862929-e242-795d-1e0b-fe7766981a5b/data/default/testBlockEvictionOnRegionMove1732398879766/da33864fc4ea86d45bff94a07435b68e/family/89e74abd0154460d901380db484b4f07, but only 0 data blocks were cached. Total data blocks for file: 1. Checking for blocks pending cache in cache writer queue. 2024-11-23T21:54:41,499 DEBUG [hfile-prefetch-1732398880564 {}] bucket.BucketCache(2318): There are still blocks pending caching for file file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/93be71cd-4dcf-2478-daf8-a2306a879141/e0862929-e242-795d-1e0b-fe7766981a5b/data/default/testBlockEvictionOnRegionMove1732398879766/da33864fc4ea86d45bff94a07435b68e/family/89e74abd0154460d901380db484b4f07. Will sleep 100ms and try the verification again. 2024-11-23T21:54:41,590 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36037 {}] master.HMaster(2410): Client=jenkins//172.17.0.2 move hri=da33864fc4ea86d45bff94a07435b68e, source=adda16083778,45163,1732398877077, destination=adda16083778,42343,1732398877237, warming up region on adda16083778,42343,1732398877237 2024-11-23T21:54:41,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36037 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-23T21:54:41,592 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36037 {}] master.HMaster(2414): Client=jenkins//172.17.0.2 move hri=da33864fc4ea86d45bff94a07435b68e, source=adda16083778,45163,1732398877077, destination=adda16083778,42343,1732398877237, running balancer 2024-11-23T21:54:41,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36037 {}] procedure2.ProcedureExecutor(1139): Stored pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1732398879766, region=da33864fc4ea86d45bff94a07435b68e, REOPEN/MOVE 2024-11-23T21:54:41,594 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1732398879766, region=da33864fc4ea86d45bff94a07435b68e, REOPEN/MOVE 2024-11-23T21:54:41,594 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-4-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47361, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-23T21:54:41,596 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=da33864fc4ea86d45bff94a07435b68e, regionState=CLOSING, regionLocation=adda16083778,45163,1732398877077 2024-11-23T21:54:41,596 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42343 {}] regionserver.RSRpcServices(2066): Warmup testBlockEvictionOnRegionMove1732398879766,,1732398879781.da33864fc4ea86d45bff94a07435b68e. 2024-11-23T21:54:41,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42343 {}] regionserver.HRegion(7855): Warmup {ENCODED => da33864fc4ea86d45bff94a07435b68e, NAME => 'testBlockEvictionOnRegionMove1732398879766,,1732398879781.da33864fc4ea86d45bff94a07435b68e.', STARTKEY => '', ENDKEY => ''} 2024-11-23T21:54:41,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42343 {}] regionserver.HRegion(898): Instantiated testBlockEvictionOnRegionMove1732398879766,,1732398879781.da33864fc4ea86d45bff94a07435b68e.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-23T21:54:41,598 DEBUG [bucket-cache-persister {}] bucket.BucketCachePersister(47): Cache is inconsistent, persisting to disk 2024-11-23T21:54:41,597 INFO [StoreOpener-da33864fc4ea86d45bff94a07435b68e-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family family of region da33864fc4ea86d45bff94a07435b68e 2024-11-23T21:54:41,598 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1487): Thread bucket-cache-persister started persisting bucket cache to file 2024-11-23T21:54:41,598 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1494): Persist in new chunked persistence format. 2024-11-23T21:54:41,598 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1673): persistToFile: before persisting backing map size: 1, fullycachedFiles size: 0, chunkSize: 10000 2024-11-23T21:54:41,599 INFO [StoreOpener-da33864fc4ea86d45bff94a07435b68e-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region da33864fc4ea86d45bff94a07435b68e columnFamilyName family 2024-11-23T21:54:41,599 DEBUG [hfile-prefetch-1732398880564 {}] bucket.BucketCache(2281): Notifying caching completed for file file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/93be71cd-4dcf-2478-daf8-a2306a879141/e0862929-e242-795d-1e0b-fe7766981a5b/data/default/testBlockEvictionOnRegionMove1732398879766/da33864fc4ea86d45bff94a07435b68e/family/89e74abd0154460d901380db484b4f07, with total blocks 2, and data blocks 1 2024-11-23T21:54:41,599 DEBUG [hfile-prefetch-1732398880564 {}] bucket.BucketCache(2285): iterating over 1 entries in the backing map 2024-11-23T21:54:41,599 DEBUG [StoreOpener-da33864fc4ea86d45bff94a07435b68e-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T21:54:41,599 DEBUG [hfile-prefetch-1732398880564 {}] bucket.BucketCache(2293): found block for file 89e74abd0154460d901380db484b4f07 in the backing map. Acquiring read lock for offset 0 2024-11-23T21:54:41,599 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-4-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1732398879766, region=da33864fc4ea86d45bff94a07435b68e, REOPEN/MOVE because future has completed 2024-11-23T21:54:41,599 DEBUG [hfile-prefetch-1732398880564 {}] bucket.BucketCache(2305): File file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/93be71cd-4dcf-2478-daf8-a2306a879141/e0862929-e242-795d-1e0b-fe7766981a5b/data/default/testBlockEvictionOnRegionMove1732398879766/da33864fc4ea86d45bff94a07435b68e/family/89e74abd0154460d901380db484b4f07 has now been fully cached. 2024-11-23T21:54:41,599 DEBUG [hfile-prefetch-1732398880564 {}] hfile.PrefetchExecutor(121): Prefetch completed for 89e74abd0154460d901380db484b4f07 2024-11-23T21:54:41,600 DEBUG [PEWorker-3 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: true 2024-11-23T21:54:41,600 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=8, ppid=7, state=RUNNABLE, hasLock=false; CloseRegionProcedure da33864fc4ea86d45bff94a07435b68e, server=adda16083778,45163,1732398877077}] 2024-11-23T21:54:41,603 DEBUG [StoreOpener-da33864fc4ea86d45bff94a07435b68e-1 {}] regionserver.StoreEngine(278): loaded file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/93be71cd-4dcf-2478-daf8-a2306a879141/e0862929-e242-795d-1e0b-fe7766981a5b/data/default/testBlockEvictionOnRegionMove1732398879766/da33864fc4ea86d45bff94a07435b68e/family/89e74abd0154460d901380db484b4f07 2024-11-23T21:54:41,604 INFO [StoreOpener-da33864fc4ea86d45bff94a07435b68e-1 {}] regionserver.HStore(327): Store=da33864fc4ea86d45bff94a07435b68e/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-23T21:54:41,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42343 {}] regionserver.HRegion(1722): Closing da33864fc4ea86d45bff94a07435b68e, disabling compactions & flushes 2024-11-23T21:54:41,604 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42343 {}] regionserver.HRegion(1755): Closing region testBlockEvictionOnRegionMove1732398879766,,1732398879781.da33864fc4ea86d45bff94a07435b68e. 2024-11-23T21:54:41,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42343 {}] regionserver.HRegion(1776): Time limited wait for close lock on testBlockEvictionOnRegionMove1732398879766,,1732398879781.da33864fc4ea86d45bff94a07435b68e. 2024-11-23T21:54:41,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42343 {}] regionserver.HRegion(1843): Acquired close lock on testBlockEvictionOnRegionMove1732398879766,,1732398879781.da33864fc4ea86d45bff94a07435b68e. after waiting 0 ms 2024-11-23T21:54:41,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42343 {}] regionserver.HRegion(1853): Updates disabled for region testBlockEvictionOnRegionMove1732398879766,,1732398879781.da33864fc4ea86d45bff94a07435b68e. 2024-11-23T21:54:41,606 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42343 {}] regionserver.HRegion(1973): Closed testBlockEvictionOnRegionMove1732398879766,,1732398879781.da33864fc4ea86d45bff94a07435b68e. 2024-11-23T21:54:41,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42343 {}] regionserver.HRegion(1676): Region close journal for da33864fc4ea86d45bff94a07435b68e: Waiting for close lock at 1732398881604Disabling compacts and flushes for region at 1732398881604Disabling writes for close at 1732398881604Writing region close event to WAL at 1732398881606 (+2 ms)Closed at 1732398881606 2024-11-23T21:54:41,616 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1680): persistToFile: after persisting backing map size: 1, fullycachedFiles size: 1 2024-11-23T21:54:41,616 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1498): PersistToFile: after persisting backing map size: 1, fullycachedFiles size: 1, file name: bucket.persistence1732398881598 2024-11-23T21:54:41,616 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1509): Thread bucket-cache-persister finished persisting bucket cache to file, renaming 2024-11-23T21:54:41,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36037 {}] procedure.ProcedureSyncWait(219): waitFor pid=7 2024-11-23T21:54:41,762 INFO [RS_CLOSE_REGION-regionserver/adda16083778:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] handler.UnassignRegionHandler(122): Close da33864fc4ea86d45bff94a07435b68e 2024-11-23T21:54:41,762 DEBUG [RS_CLOSE_REGION-regionserver/adda16083778:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: true 2024-11-23T21:54:41,764 DEBUG [RS_CLOSE_REGION-regionserver/adda16083778:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1722): Closing da33864fc4ea86d45bff94a07435b68e, disabling compactions & flushes 2024-11-23T21:54:41,764 INFO [RS_CLOSE_REGION-regionserver/adda16083778:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1755): Closing region testBlockEvictionOnRegionMove1732398879766,,1732398879781.da33864fc4ea86d45bff94a07435b68e. 2024-11-23T21:54:41,764 DEBUG [RS_CLOSE_REGION-regionserver/adda16083778:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1776): Time limited wait for close lock on testBlockEvictionOnRegionMove1732398879766,,1732398879781.da33864fc4ea86d45bff94a07435b68e. 2024-11-23T21:54:41,764 DEBUG [RS_CLOSE_REGION-regionserver/adda16083778:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1843): Acquired close lock on testBlockEvictionOnRegionMove1732398879766,,1732398879781.da33864fc4ea86d45bff94a07435b68e. after waiting 0 ms 2024-11-23T21:54:41,764 DEBUG [RS_CLOSE_REGION-regionserver/adda16083778:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1853): Updates disabled for region testBlockEvictionOnRegionMove1732398879766,,1732398879781.da33864fc4ea86d45bff94a07435b68e. 2024-11-23T21:54:41,767 DEBUG [StoreFileCloser-da33864fc4ea86d45bff94a07435b68e-family-1 {}] bucket.BucketCache(768): Removing file 89e74abd0154460d901380db484b4f07 for region da33864fc4ea86d45bff94a07435b68e 2024-11-23T21:54:41,767 DEBUG [StoreFileCloser-da33864fc4ea86d45bff94a07435b68e-family-1 {}] bucket.BucketCache(1863): found 1 blocks for file 89e74abd0154460d901380db484b4f07, starting offset: 0, end offset: 9223372036854775807 2024-11-23T21:54:41,768 DEBUG [StoreFileCloser-da33864fc4ea86d45bff94a07435b68e-family-1 {}] bucket.BucketCache(856): removed key 89e74abd0154460d901380db484b4f07_0 from back map with offset lock 2094080 in the evict process 2024-11-23T21:54:41,773 DEBUG [RS_CLOSE_REGION-regionserver/adda16083778:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/93be71cd-4dcf-2478-daf8-a2306a879141/e0862929-e242-795d-1e0b-fe7766981a5b/data/default/testBlockEvictionOnRegionMove1732398879766/da33864fc4ea86d45bff94a07435b68e/recovered.edits/9.seqid, newMaxSeqId=9, maxSeqId=1 2024-11-23T21:54:41,774 INFO [RS_CLOSE_REGION-regionserver/adda16083778:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1973): Closed testBlockEvictionOnRegionMove1732398879766,,1732398879781.da33864fc4ea86d45bff94a07435b68e. 2024-11-23T21:54:41,774 DEBUG [RS_CLOSE_REGION-regionserver/adda16083778:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1676): Region close journal for da33864fc4ea86d45bff94a07435b68e: Waiting for close lock at 1732398881763Running coprocessor pre-close hooks at 1732398881763Disabling compacts and flushes for region at 1732398881763Disabling writes for close at 1732398881764 (+1 ms)Writing region close event to WAL at 1732398881769 (+5 ms)Running coprocessor post-close hooks at 1732398881773 (+4 ms)Closed at 1732398881774 (+1 ms) 2024-11-23T21:54:41,774 DEBUG [bucket-cache-persister {}] bucket.BucketCachePersister(47): Cache is inconsistent, persisting to disk 2024-11-23T21:54:41,774 INFO [RS_CLOSE_REGION-regionserver/adda16083778:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegionServer(3302): Adding da33864fc4ea86d45bff94a07435b68e move to adda16083778,42343,1732398877237 record at close sequenceid=6 2024-11-23T21:54:41,775 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1487): Thread bucket-cache-persister started persisting bucket cache to file 2024-11-23T21:54:41,775 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1494): Persist in new chunked persistence format. 2024-11-23T21:54:41,775 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1673): persistToFile: before persisting backing map size: 0, fullycachedFiles size: 0, chunkSize: 10000 2024-11-23T21:54:41,778 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1680): persistToFile: after persisting backing map size: 0, fullycachedFiles size: 0 2024-11-23T21:54:41,778 INFO [RS_CLOSE_REGION-regionserver/adda16083778:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] handler.UnassignRegionHandler(157): Closed da33864fc4ea86d45bff94a07435b68e 2024-11-23T21:54:41,778 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1498): PersistToFile: after persisting backing map size: 0, fullycachedFiles size: 0, file name: bucket.persistence1732398881775 2024-11-23T21:54:41,778 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1509): Thread bucket-cache-persister finished persisting bucket cache to file, renaming 2024-11-23T21:54:41,779 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=da33864fc4ea86d45bff94a07435b68e, regionState=CLOSED 2024-11-23T21:54:41,782 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-4-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=8, ppid=7, state=RUNNABLE, hasLock=false; CloseRegionProcedure da33864fc4ea86d45bff94a07435b68e, server=adda16083778,45163,1732398877077 because future has completed 2024-11-23T21:54:41,787 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=8, resume processing ppid=7 2024-11-23T21:54:41,787 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=8, ppid=7, state=SUCCESS, hasLock=false; CloseRegionProcedure da33864fc4ea86d45bff94a07435b68e, server=adda16083778,45163,1732398877077 in 184 msec 2024-11-23T21:54:41,788 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1732398879766, region=da33864fc4ea86d45bff94a07435b68e, REOPEN/MOVE; state=CLOSED, location=adda16083778,42343,1732398877237; forceNewPlan=false, retain=false 2024-11-23T21:54:41,938 INFO [adda16083778:36037 {}] balancer.BaseLoadBalancer(388): Reassigned 1 regions. 1 retained the pre-restart assignment. 2024-11-23T21:54:41,939 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=da33864fc4ea86d45bff94a07435b68e, regionState=OPENING, regionLocation=adda16083778,42343,1732398877237 2024-11-23T21:54:41,943 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-4-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1732398879766, region=da33864fc4ea86d45bff94a07435b68e, REOPEN/MOVE because future has completed 2024-11-23T21:54:41,944 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=9, ppid=7, state=RUNNABLE, hasLock=false; OpenRegionProcedure da33864fc4ea86d45bff94a07435b68e, server=adda16083778,42343,1732398877237}] 2024-11-23T21:54:42,099 INFO [RS_OPEN_REGION-regionserver/adda16083778:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] handler.AssignRegionHandler(132): Open testBlockEvictionOnRegionMove1732398879766,,1732398879781.da33864fc4ea86d45bff94a07435b68e. 2024-11-23T21:54:42,099 DEBUG [RS_OPEN_REGION-regionserver/adda16083778:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(7752): Opening region: {ENCODED => da33864fc4ea86d45bff94a07435b68e, NAME => 'testBlockEvictionOnRegionMove1732398879766,,1732398879781.da33864fc4ea86d45bff94a07435b68e.', STARTKEY => '', ENDKEY => ''} 2024-11-23T21:54:42,100 DEBUG [RS_OPEN_REGION-regionserver/adda16083778:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testBlockEvictionOnRegionMove1732398879766 da33864fc4ea86d45bff94a07435b68e 2024-11-23T21:54:42,100 DEBUG [RS_OPEN_REGION-regionserver/adda16083778:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(898): Instantiated testBlockEvictionOnRegionMove1732398879766,,1732398879781.da33864fc4ea86d45bff94a07435b68e.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-23T21:54:42,100 DEBUG [RS_OPEN_REGION-regionserver/adda16083778:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(7794): checking encryption for da33864fc4ea86d45bff94a07435b68e 2024-11-23T21:54:42,100 DEBUG [RS_OPEN_REGION-regionserver/adda16083778:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(7797): checking classloading for da33864fc4ea86d45bff94a07435b68e 2024-11-23T21:54:42,101 INFO [StoreOpener-da33864fc4ea86d45bff94a07435b68e-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family family of region da33864fc4ea86d45bff94a07435b68e 2024-11-23T21:54:42,102 INFO [StoreOpener-da33864fc4ea86d45bff94a07435b68e-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region da33864fc4ea86d45bff94a07435b68e columnFamilyName family 2024-11-23T21:54:42,102 DEBUG [StoreOpener-da33864fc4ea86d45bff94a07435b68e-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T21:54:42,104 DEBUG [StoreFileOpener-da33864fc4ea86d45bff94a07435b68e-family-1 {}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/93be71cd-4dcf-2478-daf8-a2306a879141/e0862929-e242-795d-1e0b-fe7766981a5b/data/default/testBlockEvictionOnRegionMove1732398879766/da33864fc4ea86d45bff94a07435b68e/family/89e74abd0154460d901380db484b4f07, delay=932 ms 2024-11-23T21:54:42,106 DEBUG [StoreOpener-da33864fc4ea86d45bff94a07435b68e-1 {}] regionserver.StoreEngine(278): loaded file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/93be71cd-4dcf-2478-daf8-a2306a879141/e0862929-e242-795d-1e0b-fe7766981a5b/data/default/testBlockEvictionOnRegionMove1732398879766/da33864fc4ea86d45bff94a07435b68e/family/89e74abd0154460d901380db484b4f07 2024-11-23T21:54:42,106 INFO [StoreOpener-da33864fc4ea86d45bff94a07435b68e-1 {}] regionserver.HStore(327): Store=da33864fc4ea86d45bff94a07435b68e/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-23T21:54:42,106 DEBUG [RS_OPEN_REGION-regionserver/adda16083778:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1038): replaying wal for da33864fc4ea86d45bff94a07435b68e 2024-11-23T21:54:42,107 DEBUG [RS_OPEN_REGION-regionserver/adda16083778:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/93be71cd-4dcf-2478-daf8-a2306a879141/e0862929-e242-795d-1e0b-fe7766981a5b/data/default/testBlockEvictionOnRegionMove1732398879766/da33864fc4ea86d45bff94a07435b68e 2024-11-23T21:54:42,107 DEBUG [RS_OPEN_REGION-regionserver/adda16083778:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/93be71cd-4dcf-2478-daf8-a2306a879141/e0862929-e242-795d-1e0b-fe7766981a5b/data/default/testBlockEvictionOnRegionMove1732398879766/da33864fc4ea86d45bff94a07435b68e 2024-11-23T21:54:42,108 DEBUG [RS_OPEN_REGION-regionserver/adda16083778:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1048): stopping wal replay for da33864fc4ea86d45bff94a07435b68e 2024-11-23T21:54:42,108 DEBUG [RS_OPEN_REGION-regionserver/adda16083778:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1060): Cleaning up temporary data for da33864fc4ea86d45bff94a07435b68e 2024-11-23T21:54:42,112 DEBUG [RS_OPEN_REGION-regionserver/adda16083778:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1093): writing seq id for da33864fc4ea86d45bff94a07435b68e 2024-11-23T21:54:42,112 INFO [RS_OPEN_REGION-regionserver/adda16083778:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1114): Opened da33864fc4ea86d45bff94a07435b68e; next sequenceid=10; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=64457843, jitterRate=-0.03950329124927521}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-23T21:54:42,112 DEBUG [RS_OPEN_REGION-regionserver/adda16083778:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1122): Running coprocessor post-open hooks for da33864fc4ea86d45bff94a07435b68e 2024-11-23T21:54:42,113 DEBUG [RS_OPEN_REGION-regionserver/adda16083778:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1006): Region open journal for da33864fc4ea86d45bff94a07435b68e: Running coprocessor pre-open hook at 1732398882100Writing region info on filesystem at 1732398882101 (+1 ms)Initializing all the Stores at 1732398882101Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732398882101Cleaning up temporary data from old regions at 1732398882108 (+7 ms)Running coprocessor post-open hooks at 1732398882112 (+4 ms)Region opened successfully at 1732398882113 (+1 ms) 2024-11-23T21:54:42,114 INFO [RS_OPEN_REGION-regionserver/adda16083778:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegionServer(2236): Post open deploy tasks for testBlockEvictionOnRegionMove1732398879766,,1732398879781.da33864fc4ea86d45bff94a07435b68e., pid=9, masterSystemTime=1732398882097 2024-11-23T21:54:42,116 DEBUG [RS_OPEN_REGION-regionserver/adda16083778:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegionServer(2266): Finished post open deploy task for testBlockEvictionOnRegionMove1732398879766,,1732398879781.da33864fc4ea86d45bff94a07435b68e. 2024-11-23T21:54:42,116 INFO [RS_OPEN_REGION-regionserver/adda16083778:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] handler.AssignRegionHandler(153): Opened testBlockEvictionOnRegionMove1732398879766,,1732398879781.da33864fc4ea86d45bff94a07435b68e. 2024-11-23T21:54:42,117 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=da33864fc4ea86d45bff94a07435b68e, regionState=OPEN, openSeqNum=10, regionLocation=adda16083778,42343,1732398877237 2024-11-23T21:54:42,120 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-4-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=9, ppid=7, state=RUNNABLE, hasLock=false; OpenRegionProcedure da33864fc4ea86d45bff94a07435b68e, server=adda16083778,42343,1732398877237 because future has completed 2024-11-23T21:54:42,123 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=9, resume processing ppid=7 2024-11-23T21:54:42,123 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=9, ppid=7, state=SUCCESS, hasLock=false; OpenRegionProcedure da33864fc4ea86d45bff94a07435b68e, server=adda16083778,42343,1732398877237 in 177 msec 2024-11-23T21:54:42,124 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=7, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1732398879766, region=da33864fc4ea86d45bff94a07435b68e, REOPEN/MOVE in 530 msec 2024-11-23T21:54:42,147 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster 2024-11-23T21:54:42,147 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-23T21:54:42,148 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79) at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611) at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.io.hfile.TestBlockEvictionOnRegionMovement.tearDown(TestBlockEvictionOnRegionMovement.java:174) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-23T21:54:42,153 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T21:54:42,154 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T21:54:42,154 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-23T21:54:42,154 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2024-11-23T21:54:42,154 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=587878705, stopped=false 2024-11-23T21:54:42,155 INFO [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=adda16083778,36037,1732398876186 2024-11-23T21:54:42,215 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42343-0x1016afa97d20002, quorum=127.0.0.1:60787, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-23T21:54:42,215 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36037-0x1016afa97d20000, quorum=127.0.0.1:60787, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-23T21:54:42,215 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42343-0x1016afa97d20002, quorum=127.0.0.1:60787, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-23T21:54:42,215 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36037-0x1016afa97d20000, quorum=127.0.0.1:60787, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-23T21:54:42,215 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45163-0x1016afa97d20001, quorum=127.0.0.1:60787, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-23T21:54:42,215 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45163-0x1016afa97d20001, quorum=127.0.0.1:60787, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-23T21:54:42,216 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-23T21:54:42,216 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-23T21:54:42,217 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277) at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265) at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416) at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.io.hfile.TestBlockEvictionOnRegionMovement.tearDown(TestBlockEvictionOnRegionMovement.java:174) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-23T21:54:42,217 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T21:54:42,217 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:42343-0x1016afa97d20002, quorum=127.0.0.1:60787, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-23T21:54:42,218 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:36037-0x1016afa97d20000, quorum=127.0.0.1:60787, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-23T21:54:42,218 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server 'adda16083778,45163,1732398877077' ***** 2024-11-23T21:54:42,218 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-11-23T21:54:42,218 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server 'adda16083778,42343,1732398877237' ***** 2024-11-23T21:54:42,218 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:45163-0x1016afa97d20001, quorum=127.0.0.1:60787, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-23T21:54:42,218 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-11-23T21:54:42,218 INFO [RS:1;adda16083778:42343 {}] bucket.BucketCache(1738): Disabling cache 2024-11-23T21:54:42,218 INFO [RS:0;adda16083778:45163 {}] bucket.BucketCache(1738): Disabling cache 2024-11-23T21:54:42,219 INFO [RS:0;adda16083778:45163 {}] bucket.BucketCache(1762): Shutdown bucket cache: IO persistent=true; path to write=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/93be71cd-4dcf-2478-daf8-a2306a879141/bucket.persistence 2024-11-23T21:54:42,219 INFO [Time-limited test-BucketCacheWriter-0 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-0 exiting, cacheEnabled=false 2024-11-23T21:54:42,219 INFO [Time-limited test-BucketCacheWriter-0 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-0 exiting, cacheEnabled=false 2024-11-23T21:54:42,219 INFO [Time-limited test-BucketCacheWriter-2 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-2 exiting, cacheEnabled=false 2024-11-23T21:54:42,219 INFO [RS:1;adda16083778:42343 {}] bucket.BucketCache(1762): Shutdown bucket cache: IO persistent=true; path to write=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/93be71cd-4dcf-2478-daf8-a2306a879141/bucket.persistence 2024-11-23T21:54:42,219 INFO [Time-limited test-BucketCacheWriter-2 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-2 exiting, cacheEnabled=false 2024-11-23T21:54:42,219 INFO [Time-limited test-BucketCacheWriter-1 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-1 exiting, cacheEnabled=false 2024-11-23T21:54:42,219 INFO [Time-limited test-BucketCacheWriter-1 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-1 exiting, cacheEnabled=false 2024-11-23T21:54:42,219 INFO [RS:1;adda16083778:42343 {}] bucket.BucketCache(1768): Shutting down cache persister thread. 2024-11-23T21:54:42,219 INFO [RS:0;adda16083778:45163 {}] bucket.BucketCache(1768): Shutting down cache persister thread. 2024-11-23T21:54:42,219 WARN [bucket-cache-persister {}] bucket.BucketCachePersister(64): Interrupting BucketCachePersister thread. java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCachePersister.run(BucketCachePersister.java:45) ~[classes/:?] 2024-11-23T21:54:42,220 WARN [bucket-cache-persister {}] bucket.BucketCachePersister(64): Interrupting BucketCachePersister thread. java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCachePersister.run(BucketCachePersister.java:45) ~[classes/:?] 2024-11-23T21:54:42,229 DEBUG [RS:1;adda16083778:42343 {}] bucket.BucketCache(1487): Thread RS:1;adda16083778:42343 started persisting bucket cache to file 2024-11-23T21:54:42,229 DEBUG [RS:1;adda16083778:42343 {}] bucket.BucketCache(1494): Persist in new chunked persistence format. 2024-11-23T21:54:42,230 DEBUG [RS:1;adda16083778:42343 {}] bucket.BucketCache(1673): persistToFile: before persisting backing map size: 0, fullycachedFiles size: 0, chunkSize: 10000 2024-11-23T21:54:42,230 DEBUG [RS:0;adda16083778:45163 {}] bucket.BucketCache(1487): Thread RS:0;adda16083778:45163 started persisting bucket cache to file 2024-11-23T21:54:42,230 DEBUG [RS:0;adda16083778:45163 {}] bucket.BucketCache(1494): Persist in new chunked persistence format. 2024-11-23T21:54:42,230 DEBUG [RS:0;adda16083778:45163 {}] bucket.BucketCache(1673): persistToFile: before persisting backing map size: 0, fullycachedFiles size: 0, chunkSize: 10000 2024-11-23T21:54:42,232 DEBUG [RS:1;adda16083778:42343 {}] bucket.BucketCache(1680): persistToFile: after persisting backing map size: 0, fullycachedFiles size: 0 2024-11-23T21:54:42,232 DEBUG [RS:1;adda16083778:42343 {}] bucket.BucketCache(1498): PersistToFile: after persisting backing map size: 0, fullycachedFiles size: 0, file name: bucket.persistence1732398882229 2024-11-23T21:54:42,232 DEBUG [RS:1;adda16083778:42343 {}] bucket.BucketCache(1509): Thread RS:1;adda16083778:42343 finished persisting bucket cache to file, renaming 2024-11-23T21:54:42,232 DEBUG [RS:0;adda16083778:45163 {}] bucket.BucketCache(1680): persistToFile: after persisting backing map size: 0, fullycachedFiles size: 0 2024-11-23T21:54:42,233 DEBUG [RS:0;adda16083778:45163 {}] bucket.BucketCache(1498): PersistToFile: after persisting backing map size: 0, fullycachedFiles size: 0, file name: bucket.persistence1732398882230 2024-11-23T21:54:42,233 DEBUG [RS:0;adda16083778:45163 {}] bucket.BucketCache(1509): Thread RS:0;adda16083778:45163 finished persisting bucket cache to file, renaming 2024-11-23T21:54:42,233 INFO [RS:1;adda16083778:42343 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-23T21:54:42,233 INFO [RS:0;adda16083778:45163 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-23T21:54:42,233 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-23T21:54:42,233 INFO [RS:0;adda16083778:45163 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-23T21:54:42,233 INFO [RS:1;adda16083778:42343 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-23T21:54:42,233 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-23T21:54:42,233 INFO [RS:1;adda16083778:42343 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-23T21:54:42,233 INFO [RS:0;adda16083778:45163 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-23T21:54:42,234 INFO [RS:0;adda16083778:45163 {}] regionserver.HRegionServer(959): stopping server adda16083778,45163,1732398877077 2024-11-23T21:54:42,234 INFO [RS:1;adda16083778:42343 {}] regionserver.HRegionServer(3091): Received CLOSE for da33864fc4ea86d45bff94a07435b68e 2024-11-23T21:54:42,234 INFO [RS:0;adda16083778:45163 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-23T21:54:42,234 INFO [RS:0;adda16083778:45163 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;adda16083778:45163. 2024-11-23T21:54:42,234 DEBUG [RS:0;adda16083778:45163 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-23T21:54:42,234 DEBUG [RS:0;adda16083778:45163 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T21:54:42,234 INFO [RS:1;adda16083778:42343 {}] regionserver.HRegionServer(959): stopping server adda16083778,42343,1732398877237 2024-11-23T21:54:42,234 INFO [RS:1;adda16083778:42343 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-23T21:54:42,234 INFO [RS:0;adda16083778:45163 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-23T21:54:42,234 INFO [RS:0;adda16083778:45163 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-23T21:54:42,234 INFO [RS:1;adda16083778:42343 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:1;adda16083778:42343. 2024-11-23T21:54:42,234 INFO [RS:0;adda16083778:45163 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-23T21:54:42,235 DEBUG [RS:1;adda16083778:42343 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-23T21:54:42,235 INFO [RS:0;adda16083778:45163 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-11-23T21:54:42,235 DEBUG [RS:1;adda16083778:42343 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T21:54:42,235 INFO [RS_CLOSE_REGION-regionserver/adda16083778:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1662): Closing region testBlockEvictionOnRegionMove1732398879766,,1732398879781.da33864fc4ea86d45bff94a07435b68e. during a graceful stop, and cache persistence is on, so setting evict on close to false. 2024-11-23T21:54:42,235 INFO [RS:1;adda16083778:42343 {}] regionserver.HRegionServer(1321): Waiting on 1 regions to close 2024-11-23T21:54:42,235 DEBUG [RS_CLOSE_REGION-regionserver/adda16083778:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing da33864fc4ea86d45bff94a07435b68e, disabling compactions & flushes 2024-11-23T21:54:42,235 INFO [RS:0;adda16083778:45163 {}] regionserver.HRegionServer(1321): Waiting on 1 regions to close 2024-11-23T21:54:42,235 INFO [RS_CLOSE_REGION-regionserver/adda16083778:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testBlockEvictionOnRegionMove1732398879766,,1732398879781.da33864fc4ea86d45bff94a07435b68e. 2024-11-23T21:54:42,235 DEBUG [RS:1;adda16083778:42343 {}] regionserver.HRegionServer(1325): Online Regions={da33864fc4ea86d45bff94a07435b68e=testBlockEvictionOnRegionMove1732398879766,,1732398879781.da33864fc4ea86d45bff94a07435b68e.} 2024-11-23T21:54:42,235 DEBUG [RS:0;adda16083778:45163 {}] regionserver.HRegionServer(1325): Online Regions={1588230740=hbase:meta,,1.1588230740} 2024-11-23T21:54:42,235 DEBUG [RS_CLOSE_REGION-regionserver/adda16083778:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testBlockEvictionOnRegionMove1732398879766,,1732398879781.da33864fc4ea86d45bff94a07435b68e. 2024-11-23T21:54:42,235 INFO [RS_CLOSE_META-regionserver/adda16083778:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1662): Closing region hbase:meta,,1.1588230740 during a graceful stop, and cache persistence is on, so setting evict on close to false. 2024-11-23T21:54:42,235 DEBUG [RS_CLOSE_REGION-regionserver/adda16083778:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testBlockEvictionOnRegionMove1732398879766,,1732398879781.da33864fc4ea86d45bff94a07435b68e. after waiting 0 ms 2024-11-23T21:54:42,235 DEBUG [RS_CLOSE_REGION-regionserver/adda16083778:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testBlockEvictionOnRegionMove1732398879766,,1732398879781.da33864fc4ea86d45bff94a07435b68e. 2024-11-23T21:54:42,235 DEBUG [RS_CLOSE_META-regionserver/adda16083778:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-23T21:54:42,235 INFO [RS_CLOSE_META-regionserver/adda16083778:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-23T21:54:42,236 DEBUG [RS_CLOSE_META-regionserver/adda16083778:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-23T21:54:42,236 DEBUG [RS_CLOSE_META-regionserver/adda16083778:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-23T21:54:42,236 DEBUG [RS:0;adda16083778:45163 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-23T21:54:42,236 DEBUG [RS:1;adda16083778:42343 {}] regionserver.HRegionServer(1351): Waiting on da33864fc4ea86d45bff94a07435b68e 2024-11-23T21:54:42,236 DEBUG [RS_CLOSE_META-regionserver/adda16083778:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-23T21:54:42,236 INFO [RS_CLOSE_META-regionserver/adda16083778:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=3.70 KB heapSize=6.72 KB 2024-11-23T21:54:42,236 DEBUG [StoreFileCloser-da33864fc4ea86d45bff94a07435b68e-family-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/93be71cd-4dcf-2478-daf8-a2306a879141/e0862929-e242-795d-1e0b-fe7766981a5b/data/default/testBlockEvictionOnRegionMove1732398879766/da33864fc4ea86d45bff94a07435b68e/family/89e74abd0154460d901380db484b4f07 2024-11-23T21:54:42,242 DEBUG [RS_CLOSE_REGION-regionserver/adda16083778:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/93be71cd-4dcf-2478-daf8-a2306a879141/e0862929-e242-795d-1e0b-fe7766981a5b/data/default/testBlockEvictionOnRegionMove1732398879766/da33864fc4ea86d45bff94a07435b68e/recovered.edits/12.seqid, newMaxSeqId=12, maxSeqId=9 2024-11-23T21:54:42,242 INFO [RS_CLOSE_REGION-regionserver/adda16083778:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testBlockEvictionOnRegionMove1732398879766,,1732398879781.da33864fc4ea86d45bff94a07435b68e. 2024-11-23T21:54:42,243 DEBUG [RS_CLOSE_REGION-regionserver/adda16083778:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for da33864fc4ea86d45bff94a07435b68e: Waiting for close lock at 1732398882234Running coprocessor pre-close hooks at 1732398882235 (+1 ms)Disabling compacts and flushes for region at 1732398882235Disabling writes for close at 1732398882235Writing region close event to WAL at 1732398882237 (+2 ms)Running coprocessor post-close hooks at 1732398882242 (+5 ms)Closed at 1732398882242 2024-11-23T21:54:42,243 DEBUG [RS_CLOSE_REGION-regionserver/adda16083778:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testBlockEvictionOnRegionMove1732398879766,,1732398879781.da33864fc4ea86d45bff94a07435b68e. 2024-11-23T21:54:42,247 DEBUG [RS_CLOSE_META-regionserver/adda16083778:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 3b189152218c4b9184e8b6fe76f8ca69 2024-11-23T21:54:42,261 INFO [regionserver/adda16083778:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-23T21:54:42,262 DEBUG [RS_CLOSE_META-regionserver/adda16083778:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/93be71cd-4dcf-2478-daf8-a2306a879141/e0862929-e242-795d-1e0b-fe7766981a5b/data/hbase/meta/1588230740/.tmp/info/3b189152218c4b9184e8b6fe76f8ca69 is 205, key is testBlockEvictionOnRegionMove1732398879766,,1732398879781.da33864fc4ea86d45bff94a07435b68e./info:regioninfo/1732398882117/Put/seqid=0 2024-11-23T21:54:42,262 INFO [regionserver/adda16083778:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-23T21:54:42,263 INFO [RS_CLOSE_META-regionserver/adda16083778:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=3.48 KB at sequenceid=15 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/93be71cd-4dcf-2478-daf8-a2306a879141/e0862929-e242-795d-1e0b-fe7766981a5b/data/hbase/meta/1588230740/.tmp/info/3b189152218c4b9184e8b6fe76f8ca69 2024-11-23T21:54:42,271 DEBUG [RS_CLOSE_META-regionserver/adda16083778:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 8e487f51b1ae4923b02594a5645fc7ec 2024-11-23T21:54:42,283 DEBUG [RS_CLOSE_META-regionserver/adda16083778:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/93be71cd-4dcf-2478-daf8-a2306a879141/e0862929-e242-795d-1e0b-fe7766981a5b/data/hbase/meta/1588230740/.tmp/ns/8e487f51b1ae4923b02594a5645fc7ec is 43, key is default/ns:d/1732398879546/Put/seqid=0 2024-11-23T21:54:42,283 INFO [RS_CLOSE_META-regionserver/adda16083778:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=74 B at sequenceid=15 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/93be71cd-4dcf-2478-daf8-a2306a879141/e0862929-e242-795d-1e0b-fe7766981a5b/data/hbase/meta/1588230740/.tmp/ns/8e487f51b1ae4923b02594a5645fc7ec 2024-11-23T21:54:42,291 DEBUG [RS_CLOSE_META-regionserver/adda16083778:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 3c09299f584c4afdb76066d4938f22a8 2024-11-23T21:54:42,303 DEBUG [RS_CLOSE_META-regionserver/adda16083778:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/93be71cd-4dcf-2478-daf8-a2306a879141/e0862929-e242-795d-1e0b-fe7766981a5b/data/hbase/meta/1588230740/.tmp/table/3c09299f584c4afdb76066d4938f22a8 is 78, key is testBlockEvictionOnRegionMove1732398879766/table:state/1732398880204/Put/seqid=0 2024-11-23T21:54:42,304 INFO [RS_CLOSE_META-regionserver/adda16083778:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=148 B at sequenceid=15 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/93be71cd-4dcf-2478-daf8-a2306a879141/e0862929-e242-795d-1e0b-fe7766981a5b/data/hbase/meta/1588230740/.tmp/table/3c09299f584c4afdb76066d4938f22a8 2024-11-23T21:54:42,307 DEBUG [RS_CLOSE_META-regionserver/adda16083778:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/93be71cd-4dcf-2478-daf8-a2306a879141/e0862929-e242-795d-1e0b-fe7766981a5b/data/hbase/meta/1588230740/.tmp/info/3b189152218c4b9184e8b6fe76f8ca69 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/93be71cd-4dcf-2478-daf8-a2306a879141/e0862929-e242-795d-1e0b-fe7766981a5b/data/hbase/meta/1588230740/info/3b189152218c4b9184e8b6fe76f8ca69 2024-11-23T21:54:42,308 DEBUG [RS_CLOSE_META-regionserver/adda16083778:0-0 {event_type=M_RS_CLOSE_META}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/93be71cd-4dcf-2478-daf8-a2306a879141/e0862929-e242-795d-1e0b-fe7766981a5b/data/hbase/meta/1588230740/info/3b189152218c4b9184e8b6fe76f8ca69, delay=994 ms 2024-11-23T21:54:42,310 INFO [RS_CLOSE_META-regionserver/adda16083778:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/93be71cd-4dcf-2478-daf8-a2306a879141/e0862929-e242-795d-1e0b-fe7766981a5b/data/hbase/meta/1588230740/info/3b189152218c4b9184e8b6fe76f8ca69, entries=15, sequenceid=15, filesize=7.6 K 2024-11-23T21:54:42,310 DEBUG [RS_CLOSE_META-regionserver/adda16083778:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/93be71cd-4dcf-2478-daf8-a2306a879141/e0862929-e242-795d-1e0b-fe7766981a5b/data/hbase/meta/1588230740/.tmp/ns/8e487f51b1ae4923b02594a5645fc7ec as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/93be71cd-4dcf-2478-daf8-a2306a879141/e0862929-e242-795d-1e0b-fe7766981a5b/data/hbase/meta/1588230740/ns/8e487f51b1ae4923b02594a5645fc7ec 2024-11-23T21:54:42,312 DEBUG [RS_CLOSE_META-regionserver/adda16083778:0-0 {event_type=M_RS_CLOSE_META}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/93be71cd-4dcf-2478-daf8-a2306a879141/e0862929-e242-795d-1e0b-fe7766981a5b/data/hbase/meta/1588230740/ns/8e487f51b1ae4923b02594a5645fc7ec, delay=972 ms 2024-11-23T21:54:42,313 INFO [RS_CLOSE_META-regionserver/adda16083778:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/93be71cd-4dcf-2478-daf8-a2306a879141/e0862929-e242-795d-1e0b-fe7766981a5b/data/hbase/meta/1588230740/ns/8e487f51b1ae4923b02594a5645fc7ec, entries=2, sequenceid=15, filesize=5.0 K 2024-11-23T21:54:42,314 DEBUG [RS_CLOSE_META-regionserver/adda16083778:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/93be71cd-4dcf-2478-daf8-a2306a879141/e0862929-e242-795d-1e0b-fe7766981a5b/data/hbase/meta/1588230740/.tmp/table/3c09299f584c4afdb76066d4938f22a8 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/93be71cd-4dcf-2478-daf8-a2306a879141/e0862929-e242-795d-1e0b-fe7766981a5b/data/hbase/meta/1588230740/table/3c09299f584c4afdb76066d4938f22a8 2024-11-23T21:54:42,316 DEBUG [RS_CLOSE_META-regionserver/adda16083778:0-0 {event_type=M_RS_CLOSE_META}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/93be71cd-4dcf-2478-daf8-a2306a879141/e0862929-e242-795d-1e0b-fe7766981a5b/data/hbase/meta/1588230740/table/3c09299f584c4afdb76066d4938f22a8, delay=962 ms 2024-11-23T21:54:42,318 INFO [RS_CLOSE_META-regionserver/adda16083778:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/93be71cd-4dcf-2478-daf8-a2306a879141/e0862929-e242-795d-1e0b-fe7766981a5b/data/hbase/meta/1588230740/table/3c09299f584c4afdb76066d4938f22a8, entries=2, sequenceid=15, filesize=5.3 K 2024-11-23T21:54:42,319 INFO [RS_CLOSE_META-regionserver/adda16083778:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~3.70 KB/3789, heapSize ~6.42 KB/6576, currentSize=0 B/0 for 1588230740 in 83ms, sequenceid=15, compaction requested=false 2024-11-23T21:54:42,319 DEBUG [RS_CLOSE_META-regionserver/adda16083778:0-0 {event_type=M_RS_CLOSE_META}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta' 2024-11-23T21:54:42,320 DEBUG [StoreFileCloser-1588230740-info-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/93be71cd-4dcf-2478-daf8-a2306a879141/e0862929-e242-795d-1e0b-fe7766981a5b/data/hbase/meta/1588230740/info/3b189152218c4b9184e8b6fe76f8ca69 2024-11-23T21:54:42,321 DEBUG [StoreFileCloser-1588230740-ns-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/93be71cd-4dcf-2478-daf8-a2306a879141/e0862929-e242-795d-1e0b-fe7766981a5b/data/hbase/meta/1588230740/ns/8e487f51b1ae4923b02594a5645fc7ec 2024-11-23T21:54:42,321 DEBUG [StoreFileCloser-1588230740-table-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/93be71cd-4dcf-2478-daf8-a2306a879141/e0862929-e242-795d-1e0b-fe7766981a5b/data/hbase/meta/1588230740/table/3c09299f584c4afdb76066d4938f22a8 2024-11-23T21:54:42,325 DEBUG [RS_CLOSE_META-regionserver/adda16083778:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/93be71cd-4dcf-2478-daf8-a2306a879141/e0862929-e242-795d-1e0b-fe7766981a5b/data/hbase/meta/1588230740/recovered.edits/18.seqid, newMaxSeqId=18, maxSeqId=1 2024-11-23T21:54:42,325 DEBUG [RS_CLOSE_META-regionserver/adda16083778:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-23T21:54:42,326 INFO [RS_CLOSE_META-regionserver/adda16083778:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-23T21:54:42,326 DEBUG [RS_CLOSE_META-regionserver/adda16083778:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1732398882235Running coprocessor pre-close hooks at 1732398882235Disabling compacts and flushes for region at 1732398882235Disabling writes for close at 1732398882236 (+1 ms)Obtaining lock to block concurrent updates at 1732398882236Preparing flush snapshotting stores in 1588230740 at 1732398882236Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=3789, getHeapSize=6816, getOffHeapSize=0, getCellsCount=27 at 1732398882237 (+1 ms)Flushing stores of hbase:meta,,1.1588230740 at 1732398882237Flushing 1588230740/info: creating writer at 1732398882237Flushing 1588230740/info: appending metadata at 1732398882262 (+25 ms)Flushing 1588230740/info: closing flushed file at 1732398882262Flushing 1588230740/ns: creating writer at 1732398882266 (+4 ms)Flushing 1588230740/ns: appending metadata at 1732398882282 (+16 ms)Flushing 1588230740/ns: closing flushed file at 1732398882282Flushing 1588230740/table: creating writer at 1732398882286 (+4 ms)Flushing 1588230740/table: appending metadata at 1732398882302 (+16 ms)Flushing 1588230740/table: closing flushed file at 1732398882302Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@6ff90d04: reopening flushed file at 1732398882307 (+5 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@1dc124f6: reopening flushed file at 1732398882310 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@27fd9fe6: reopening flushed file at 1732398882314 (+4 ms)Finished flush of dataSize ~3.70 KB/3789, heapSize ~6.42 KB/6576, currentSize=0 B/0 for 1588230740 in 83ms, sequenceid=15, compaction requested=false at 1732398882319 (+5 ms)Writing region close event to WAL at 1732398882322 (+3 ms)Running coprocessor post-close hooks at 1732398882325 (+3 ms)Closed at 1732398882326 (+1 ms) 2024-11-23T21:54:42,326 DEBUG [RS_CLOSE_META-regionserver/adda16083778:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-11-23T21:54:42,436 INFO [RS:0;adda16083778:45163 {}] regionserver.HRegionServer(976): stopping server adda16083778,45163,1732398877077; all regions closed. 2024-11-23T21:54:42,436 INFO [RS:1;adda16083778:42343 {}] regionserver.HRegionServer(976): stopping server adda16083778,42343,1732398877237; all regions closed. 2024-11-23T21:54:42,445 DEBUG [RS:1;adda16083778:42343 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/93be71cd-4dcf-2478-daf8-a2306a879141/e0862929-e242-795d-1e0b-fe7766981a5b/oldWALs 2024-11-23T21:54:42,445 DEBUG [RS:0;adda16083778:45163 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/93be71cd-4dcf-2478-daf8-a2306a879141/e0862929-e242-795d-1e0b-fe7766981a5b/oldWALs 2024-11-23T21:54:42,445 INFO [RS:1;adda16083778:42343 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL adda16083778%2C42343%2C1732398877237:(num 1732398878832) 2024-11-23T21:54:42,445 INFO [RS:0;adda16083778:45163 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL adda16083778%2C45163%2C1732398877077.meta:.meta(num 1732398879370) 2024-11-23T21:54:42,445 DEBUG [RS:1;adda16083778:42343 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T21:54:42,445 INFO [RS:1;adda16083778:42343 {}] regionserver.LeaseManager(133): Closed leases 2024-11-23T21:54:42,446 INFO [RS:1;adda16083778:42343 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-23T21:54:42,446 INFO [RS:1;adda16083778:42343 {}] hbase.ChoreService(370): Chore service for: regionserver/adda16083778:0 had [ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-11-23T21:54:42,447 INFO [RS:1;adda16083778:42343 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-23T21:54:42,447 INFO [RS:1;adda16083778:42343 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-23T21:54:42,447 INFO [RS:1;adda16083778:42343 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-23T21:54:42,447 INFO [regionserver/adda16083778:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-23T21:54:42,447 INFO [RS:1;adda16083778:42343 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-23T21:54:42,447 INFO [RS:1;adda16083778:42343 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:42343 2024-11-23T21:54:42,448 DEBUG [RS:0;adda16083778:45163 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/93be71cd-4dcf-2478-daf8-a2306a879141/e0862929-e242-795d-1e0b-fe7766981a5b/oldWALs 2024-11-23T21:54:42,448 INFO [RS:0;adda16083778:45163 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL adda16083778%2C45163%2C1732398877077:(num 1732398878832) 2024-11-23T21:54:42,448 DEBUG [RS:0;adda16083778:45163 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T21:54:42,449 INFO [RS:0;adda16083778:45163 {}] regionserver.LeaseManager(133): Closed leases 2024-11-23T21:54:42,449 INFO [RS:0;adda16083778:45163 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-23T21:54:42,449 INFO [RS:0;adda16083778:45163 {}] hbase.ChoreService(370): Chore service for: regionserver/adda16083778:0 had [ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS] on shutdown 2024-11-23T21:54:42,449 INFO [RS:0;adda16083778:45163 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-23T21:54:42,449 INFO [regionserver/adda16083778:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-23T21:54:42,450 INFO [RS:0;adda16083778:45163 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:45163 2024-11-23T21:54:42,473 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36037-0x1016afa97d20000, quorum=127.0.0.1:60787, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-23T21:54:42,473 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45163-0x1016afa97d20001, quorum=127.0.0.1:60787, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/adda16083778,45163,1732398877077 2024-11-23T21:54:42,473 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42343-0x1016afa97d20002, quorum=127.0.0.1:60787, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/adda16083778,42343,1732398877237 2024-11-23T21:54:42,473 INFO [RS:0;adda16083778:45163 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-23T21:54:42,473 INFO [RS:1;adda16083778:42343 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-23T21:54:42,481 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [adda16083778,45163,1732398877077] 2024-11-23T21:54:42,531 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/adda16083778,45163,1732398877077 already deleted, retry=false 2024-11-23T21:54:42,531 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; adda16083778,45163,1732398877077 expired; onlineServers=1 2024-11-23T21:54:42,531 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [adda16083778,42343,1732398877237] 2024-11-23T21:54:42,539 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/adda16083778,42343,1732398877237 already deleted, retry=false 2024-11-23T21:54:42,539 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; adda16083778,42343,1732398877237 expired; onlineServers=0 2024-11-23T21:54:42,540 INFO [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master 'adda16083778,36037,1732398876186' ***** 2024-11-23T21:54:42,540 INFO [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0 2024-11-23T21:54:42,540 INFO [M:0;adda16083778:36037 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-23T21:54:42,540 INFO [M:0;adda16083778:36037 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-23T21:54:42,540 DEBUG [M:0;adda16083778:36037 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2024-11-23T21:54:42,541 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2024-11-23T21:54:42,541 DEBUG [M:0;adda16083778:36037 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2024-11-23T21:54:42,541 DEBUG [master/adda16083778:0:becomeActiveMaster-HFileCleaner.small.0-1732398878345 {}] cleaner.HFileCleaner(306): Exit Thread[master/adda16083778:0:becomeActiveMaster-HFileCleaner.small.0-1732398878345,5,FailOnTimeoutGroup] 2024-11-23T21:54:42,541 DEBUG [master/adda16083778:0:becomeActiveMaster-HFileCleaner.large.0-1732398878344 {}] cleaner.HFileCleaner(306): Exit Thread[master/adda16083778:0:becomeActiveMaster-HFileCleaner.large.0-1732398878344,5,FailOnTimeoutGroup] 2024-11-23T21:54:42,541 INFO [M:0;adda16083778:36037 {}] hbase.ChoreService(370): Chore service for: master/adda16083778:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown 2024-11-23T21:54:42,542 INFO [M:0;adda16083778:36037 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-23T21:54:42,542 DEBUG [M:0;adda16083778:36037 {}] master.HMaster(1795): Stopping service threads 2024-11-23T21:54:42,542 INFO [M:0;adda16083778:36037 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2024-11-23T21:54:42,542 INFO [M:0;adda16083778:36037 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-23T21:54:42,543 INFO [M:0;adda16083778:36037 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-23T21:54:42,543 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2024-11-23T21:54:42,548 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36037-0x1016afa97d20000, quorum=127.0.0.1:60787, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-11-23T21:54:42,548 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36037-0x1016afa97d20000, quorum=127.0.0.1:60787, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-23T21:54:42,548 DEBUG [M:0;adda16083778:36037 {}] zookeeper.ZKUtil(347): master:36037-0x1016afa97d20000, quorum=127.0.0.1:60787, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error) 2024-11-23T21:54:42,548 WARN [M:0;adda16083778:36037 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null 2024-11-23T21:54:42,549 INFO [M:0;adda16083778:36037 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/93be71cd-4dcf-2478-daf8-a2306a879141/e0862929-e242-795d-1e0b-fe7766981a5b/.lastflushedseqids 2024-11-23T21:54:42,559 INFO [M:0;adda16083778:36037 {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-23T21:54:42,559 INFO [M:0;adda16083778:36037 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2024-11-23T21:54:42,560 DEBUG [M:0;adda16083778:36037 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-23T21:54:42,560 INFO [M:0;adda16083778:36037 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-23T21:54:42,560 DEBUG [M:0;adda16083778:36037 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-23T21:54:42,560 DEBUG [M:0;adda16083778:36037 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-23T21:54:42,560 DEBUG [M:0;adda16083778:36037 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-23T21:54:42,560 INFO [M:0;adda16083778:36037 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=34.69 KB heapSize=43.02 KB 2024-11-23T21:54:42,569 DEBUG [M:0;adda16083778:36037 {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 9c9cd1c3bc1446e0970dba3a2acff917 2024-11-23T21:54:42,581 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45163-0x1016afa97d20001, quorum=127.0.0.1:60787, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-23T21:54:42,581 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42343-0x1016afa97d20002, quorum=127.0.0.1:60787, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-23T21:54:42,582 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42343-0x1016afa97d20002, quorum=127.0.0.1:60787, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-23T21:54:42,582 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45163-0x1016afa97d20001, quorum=127.0.0.1:60787, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-23T21:54:42,582 INFO [RS:1;adda16083778:42343 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-23T21:54:42,582 INFO [RS:0;adda16083778:45163 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-23T21:54:42,582 INFO [RS:0;adda16083778:45163 {}] regionserver.HRegionServer(1031): Exiting; stopping=adda16083778,45163,1732398877077; zookeeper connection closed. 2024-11-23T21:54:42,582 INFO [RS:1;adda16083778:42343 {}] regionserver.HRegionServer(1031): Exiting; stopping=adda16083778,42343,1732398877237; zookeeper connection closed. 2024-11-23T21:54:42,582 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@4f4ed395 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@4f4ed395 2024-11-23T21:54:42,582 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@57015d58 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@57015d58 2024-11-23T21:54:42,582 DEBUG [M:0;adda16083778:36037 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/93be71cd-4dcf-2478-daf8-a2306a879141/e0862929-e242-795d-1e0b-fe7766981a5b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/9c9cd1c3bc1446e0970dba3a2acff917 is 82, key is hbase:meta,,1/info:regioninfo/1732398879430/Put/seqid=0 2024-11-23T21:54:42,583 INFO [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 2 regionserver(s) complete 2024-11-23T21:54:42,583 INFO [M:0;adda16083778:36037 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=504 B at sequenceid=84 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/93be71cd-4dcf-2478-daf8-a2306a879141/e0862929-e242-795d-1e0b-fe7766981a5b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/9c9cd1c3bc1446e0970dba3a2acff917 2024-11-23T21:54:42,591 DEBUG [M:0;adda16083778:36037 {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 19edca92b834424cad209bc9d09fbb93 2024-11-23T21:54:42,603 DEBUG [M:0;adda16083778:36037 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/93be71cd-4dcf-2478-daf8-a2306a879141/e0862929-e242-795d-1e0b-fe7766981a5b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/19edca92b834424cad209bc9d09fbb93 is 779, key is \x00\x00\x00\x00\x00\x00\x00\x04/proc:d/1732398880210/Put/seqid=0 2024-11-23T21:54:42,604 INFO [M:0;adda16083778:36037 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=34.07 KB at sequenceid=84 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/93be71cd-4dcf-2478-daf8-a2306a879141/e0862929-e242-795d-1e0b-fe7766981a5b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/19edca92b834424cad209bc9d09fbb93 2024-11-23T21:54:42,610 DEBUG [M:0;adda16083778:36037 {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 7cd3a0293b414a0da87ce7ed5a634690 2024-11-23T21:54:42,621 DEBUG [M:0;adda16083778:36037 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/93be71cd-4dcf-2478-daf8-a2306a879141/e0862929-e242-795d-1e0b-fe7766981a5b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/7cd3a0293b414a0da87ce7ed5a634690 is 69, key is adda16083778,42343,1732398877237/rs:state/1732398878499/Put/seqid=0 2024-11-23T21:54:42,622 INFO [M:0;adda16083778:36037 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=130 B at sequenceid=84 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/93be71cd-4dcf-2478-daf8-a2306a879141/e0862929-e242-795d-1e0b-fe7766981a5b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/7cd3a0293b414a0da87ce7ed5a634690 2024-11-23T21:54:42,624 DEBUG [M:0;adda16083778:36037 {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/93be71cd-4dcf-2478-daf8-a2306a879141/e0862929-e242-795d-1e0b-fe7766981a5b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/9c9cd1c3bc1446e0970dba3a2acff917 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/93be71cd-4dcf-2478-daf8-a2306a879141/e0862929-e242-795d-1e0b-fe7766981a5b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/9c9cd1c3bc1446e0970dba3a2acff917 2024-11-23T21:54:42,626 INFO [M:0;adda16083778:36037 {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/93be71cd-4dcf-2478-daf8-a2306a879141/e0862929-e242-795d-1e0b-fe7766981a5b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/9c9cd1c3bc1446e0970dba3a2acff917, entries=8, sequenceid=84, filesize=5.5 K 2024-11-23T21:54:42,627 DEBUG [M:0;adda16083778:36037 {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/93be71cd-4dcf-2478-daf8-a2306a879141/e0862929-e242-795d-1e0b-fe7766981a5b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/19edca92b834424cad209bc9d09fbb93 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/93be71cd-4dcf-2478-daf8-a2306a879141/e0862929-e242-795d-1e0b-fe7766981a5b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/19edca92b834424cad209bc9d09fbb93 2024-11-23T21:54:42,629 INFO [M:0;adda16083778:36037 {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/93be71cd-4dcf-2478-daf8-a2306a879141/e0862929-e242-795d-1e0b-fe7766981a5b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/19edca92b834424cad209bc9d09fbb93, entries=9, sequenceid=84, filesize=6.5 K 2024-11-23T21:54:42,629 DEBUG [M:0;adda16083778:36037 {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/93be71cd-4dcf-2478-daf8-a2306a879141/e0862929-e242-795d-1e0b-fe7766981a5b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/7cd3a0293b414a0da87ce7ed5a634690 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/93be71cd-4dcf-2478-daf8-a2306a879141/e0862929-e242-795d-1e0b-fe7766981a5b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/7cd3a0293b414a0da87ce7ed5a634690 2024-11-23T21:54:42,631 INFO [M:0;adda16083778:36037 {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/93be71cd-4dcf-2478-daf8-a2306a879141/e0862929-e242-795d-1e0b-fe7766981a5b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/7cd3a0293b414a0da87ce7ed5a634690, entries=2, sequenceid=84, filesize=5.1 K 2024-11-23T21:54:42,632 INFO [M:0;adda16083778:36037 {}] regionserver.HRegion(3140): Finished flush of dataSize ~34.69 KB/35518, heapSize ~42.73 KB/43752, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 72ms, sequenceid=84, compaction requested=false 2024-11-23T21:54:42,634 INFO [M:0;adda16083778:36037 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-23T21:54:42,634 DEBUG [M:0;adda16083778:36037 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1732398882560Disabling compacts and flushes for region at 1732398882560Disabling writes for close at 1732398882560Obtaining lock to block concurrent updates at 1732398882560Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1732398882560Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=35518, getHeapSize=43992, getOffHeapSize=0, getCellsCount=99 at 1732398882561 (+1 ms)Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1732398882561Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1732398882561Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1732398882582 (+21 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1732398882582Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1732398882585 (+3 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1732398882603 (+18 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1732398882603Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1732398882605 (+2 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1732398882621 (+16 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1732398882621Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@37791a09: reopening flushed file at 1732398882624 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@51161dc7: reopening flushed file at 1732398882626 (+2 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@48d684ca: reopening flushed file at 1732398882629 (+3 ms)Finished flush of dataSize ~34.69 KB/35518, heapSize ~42.73 KB/43752, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 72ms, sequenceid=84, compaction requested=false at 1732398882632 (+3 ms)Writing region close event to WAL at 1732398882634 (+2 ms)Closed at 1732398882634 2024-11-23T21:54:42,636 INFO [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-23T21:54:42,636 INFO [M:0;adda16083778:36037 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down. 2024-11-23T21:54:42,636 INFO [M:0;adda16083778:36037 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:36037 2024-11-23T21:54:42,636 INFO [M:0;adda16083778:36037 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-23T21:54:42,772 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36037-0x1016afa97d20000, quorum=127.0.0.1:60787, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-23T21:54:42,772 INFO [M:0;adda16083778:36037 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-23T21:54:42,772 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36037-0x1016afa97d20000, quorum=127.0.0.1:60787, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-23T21:54:42,781 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers 2024-11-23T21:54:42,801 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down 2024-11-23T21:54:42,801 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-23T21:54:42,809 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: io.hfile.TestBlockEvictionOnRegionMovement#testBlockEvictionOnRegionMove Thread=50 (was 12) Potentially hanging thread: master/adda16083778:0:becomeActiveMaster-MemStoreChunkPool Statistics java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HBase-Metrics2-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: regionserver/adda16083778:0.procedureResultReporter java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.RemoteProcedureResultReporter.run(RemoteProcedureResultReporter.java:75) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-4-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-4-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: region-location-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1170) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: hfile-prefetch-1732398882104 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1177) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Async-Client-Retry-Timer-pool-0 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.waitForNextTick(HashedWheelTimer.java:598) app//org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:494) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: org.apache.hadoop.fs.FileSystem$Statistics$StatisticsDataReferenceCleaner java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:155) java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:176) app//org.apache.hadoop.fs.FileSystem$Statistics$StatisticsDataReferenceCleaner.run(FileSystem.java:4171) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HMaster-EventLoopGroup-1-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: SnapshotHandlerChoreCleaner java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-4 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-4-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: master/adda16083778:0:becomeActiveMaster-MemStoreChunkPool Statistics java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Idle-Rpc-Conn-Sweeper-pool-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: regionserver/adda16083778:0.procedureResultReporter java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.RemoteProcedureResultReporter.run(RemoteProcedureResultReporter.java:75) Potentially hanging thread: RpcClient-timer-pool-0 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.waitForNextTick(HashedWheelTimer.java:598) app//org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:494) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Timer for 'HBase' metrics system java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.util.TimerThread.mainLoop(Timer.java:563) java.base@17.0.11/java.util.TimerThread.run(Timer.java:516) Potentially hanging thread: hfile-prefetch-1732398882312 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1177) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HMaster-EventLoopGroup-1-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ForkJoinPool.commonPool-worker-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkUntil(LockSupport.java:410) java.base@17.0.11/java.util.concurrent.ForkJoinPool.awaitWork(ForkJoinPool.java:1726) java.base@17.0.11/java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1623) java.base@17.0.11/java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:165) Potentially hanging thread: hfile-prefetch-1732398882308 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1177) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ForkJoinPool.commonPool-worker-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.ForkJoinPool.awaitWork(ForkJoinPool.java:1724) java.base@17.0.11/java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1623) java.base@17.0.11/java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:165) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-3-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-3-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: SessionTracker java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.zookeeper.server.SessionTrackerImpl.run(SessionTrackerImpl.java:163) Potentially hanging thread: HMaster-EventLoopGroup-1-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: region-location-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1170) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Time-limited test.named-queue-events-pool-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) app//com.lmax.disruptor.BlockingWaitStrategy.waitFor(BlockingWaitStrategy.java:47) app//com.lmax.disruptor.ProcessingSequenceBarrier.waitFor(ProcessingSequenceBarrier.java:56) app//com.lmax.disruptor.BatchEventProcessor.processEvents(BatchEventProcessor.java:159) app//com.lmax.disruptor.BatchEventProcessor.run(BatchEventProcessor.java:125) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: hfile-prefetch-1732398880564 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Monitor thread for TaskMonitor java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hbase.monitoring.TaskMonitor$MonitorRunnable.run(TaskMonitor.java:325) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ForkJoinPool.commonPool-worker-3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.ForkJoinPool.awaitWork(ForkJoinPool.java:1724) java.base@17.0.11/java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1623) java.base@17.0.11/java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:165) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: weak-ref-cleaner-strictcontextstorage java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:155) java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:176) app//io.opentelemetry.context.StrictContextStorage$PendingScopes.run(StrictContextStorage.java:269) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-3-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=346 (was 287) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=418 (was 445), ProcessCount=11 (was 11), AvailableMemoryMB=8509 (was 8883) 2024-11-23T21:54:42,814 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: io.hfile.TestBlockEvictionOnRegionMovement#testBlockEvictionOnGracefulStop Thread=50, OpenFileDescriptor=346, MaxFileDescriptor=1048576, SystemLoadAverage=418, ProcessCount=11, AvailableMemoryMB=8509 2024-11-23T21:54:42,814 INFO [Time-limited test {}] hbase.HBaseTestingUtil(401): System.getProperty("hadoop.log.dir") already set to: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/93be71cd-4dcf-2478-daf8-a2306a879141/hadoop-log-dir so I do NOT create it in target/test-data/d246ebad-447b-0ccb-984f-a43056e9385e 2024-11-23T21:54:42,814 INFO [Time-limited test {}] hbase.HBaseTestingUtil(401): System.getProperty("hadoop.tmp.dir") already set to: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/93be71cd-4dcf-2478-daf8-a2306a879141/hadoop-tmp-dir so I do NOT create it in target/test-data/d246ebad-447b-0ccb-984f-a43056e9385e 2024-11-23T21:54:42,814 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-23T21:54:42,817 INFO [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d246ebad-447b-0ccb-984f-a43056e9385e/cluster_efbce95b-6058-acd6-b356-fe5051ddb46b, deleteOnExit=true 2024-11-23T21:54:42,820 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d246ebad-447b-0ccb-984f-a43056e9385e/cluster_efbce95b-6058-acd6-b356-fe5051ddb46b/zookeeper_0, clientPort=51781, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d246ebad-447b-0ccb-984f-a43056e9385e/cluster_efbce95b-6058-acd6-b356-fe5051ddb46b/zookeeper_0/version-2, dataDirSize=457 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d246ebad-447b-0ccb-984f-a43056e9385e/cluster_efbce95b-6058-acd6-b356-fe5051ddb46b/zookeeper_0/version-2, dataLogSize=457 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0 2024-11-23T21:54:42,823 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=51781 2024-11-23T21:54:42,832 INFO [Time-limited test {}] util.FSUtils(489): Created version file at /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/93be71cd-4dcf-2478-daf8-a2306a879141/e0862929-e242-795d-1e0b-fe7766981a5b with version=8 2024-11-23T21:54:42,832 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1141): The hbase.fs.tmp.dir is set to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/93be71cd-4dcf-2478-daf8-a2306a879141/e0862929-e242-795d-1e0b-fe7766981a5b/hbase-staging 2024-11-23T21:54:42,834 INFO [Time-limited test {}] client.ConnectionUtils(128): master/adda16083778:0 server-side Connection retries=45 2024-11-23T21:54:42,834 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-23T21:54:42,834 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-23T21:54:42,835 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-23T21:54:42,835 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-23T21:54:42,835 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-23T21:54:42,835 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService 2024-11-23T21:54:42,835 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-23T21:54:42,836 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:33953 2024-11-23T21:54:42,838 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:33953 connecting to ZooKeeper ensemble=127.0.0.1:51781 2024-11-23T21:54:42,876 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:339530x0, quorum=127.0.0.1:51781, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-23T21:54:42,876 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:33953-0x1016afab2c80000 connected 2024-11-23T21:54:42,959 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-23T21:54:42,959 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-23T21:54:42,960 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:33953-0x1016afab2c80000, quorum=127.0.0.1:51781, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-23T21:54:42,960 INFO [Time-limited test {}] master.HMaster(525): hbase.rootdir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/93be71cd-4dcf-2478-daf8-a2306a879141/e0862929-e242-795d-1e0b-fe7766981a5b, hbase.cluster.distributed=false 2024-11-23T21:54:42,962 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:33953-0x1016afab2c80000, quorum=127.0.0.1:51781, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-23T21:54:42,963 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=33953 2024-11-23T21:54:42,963 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=33953 2024-11-23T21:54:42,963 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=33953 2024-11-23T21:54:42,963 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=33953 2024-11-23T21:54:42,964 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=33953 2024-11-23T21:54:42,981 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/adda16083778:0 server-side Connection retries=45 2024-11-23T21:54:42,982 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-23T21:54:42,982 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-23T21:54:42,982 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-23T21:54:42,982 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-23T21:54:42,982 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-23T21:54:42,982 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-23T21:54:42,982 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-23T21:54:42,983 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:37831 2024-11-23T21:54:42,984 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:37831 connecting to ZooKeeper ensemble=127.0.0.1:51781 2024-11-23T21:54:42,984 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-23T21:54:42,984 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-23T21:54:42,992 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:378310x0, quorum=127.0.0.1:51781, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-23T21:54:42,993 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:37831-0x1016afab2c80001 connected 2024-11-23T21:54:42,993 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:37831-0x1016afab2c80001, quorum=127.0.0.1:51781, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-23T21:54:42,993 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-23T21:54:42,994 INFO [Time-limited test {}] bucket.FileIOEngine(99): Allocating cache 400 MB, on the path:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d246ebad-447b-0ccb-984f-a43056e9385e/bucket.cache 2024-11-23T21:54:42,994 INFO [Time-limited test {}] bucket.BucketCache(364): Instantiating BucketCache with acceptableFactor: 0.95, minFactor: 0.85, extraFreeFactor: 0.1, singleFactor: 0.25, multiFactor: 0.5, memoryFactor: 0.25, useStrongRef: false 2024-11-23T21:54:42,994 INFO [Time-limited test {}] bucket.BucketCache(478): Starting BucketCachePersisterThread 2024-11-23T21:54:42,994 INFO [Time-limited test {}] bucket.BucketCachePersister(38): BucketCachePersister started with interval: 100 2024-11-23T21:54:42,994 INFO [Thread-118 {}] bucket.BucketCache(1530): Started retrieving bucket cache from file 2024-11-23T21:54:42,994 WARN [Thread-118 {}] bucket.BucketCache(1533): Persistence file missing! It's ok if it's first run after enabling persistent cache. 2024-11-23T21:54:42,994 INFO [Time-limited test {}] bucket.BucketCache(403): Started bucket cache; ioengine=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d246ebad-447b-0ccb-984f-a43056e9385e/bucket.cache, capacity=400 MB, blockSize=64 KB, writerThreadNum=3, writerQLen=64, persistencePath=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d246ebad-447b-0ccb-984f-a43056e9385e/bucket.persistence, bucketAllocator=org.apache.hadoop.hbase.io.hfile.bucket.BucketAllocator 2024-11-23T21:54:42,995 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-23T21:54:42,995 INFO [Thread-118 {}] bucket.BucketAllocator(343): Cache totalSize=418148352, buckets=199, bucket capacity=2101248=(4*525312)=(FEWEST_ITEMS_IN_BUCKET*(largest configured bucketcache size)) 2024-11-23T21:54:42,995 INFO [Thread-118 {}] bucket.BucketCache(413): Persistent bucket cache recovery from /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d246ebad-447b-0ccb-984f-a43056e9385e/bucket.persistence is complete. 2024-11-23T21:54:42,996 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:37831-0x1016afab2c80001, quorum=127.0.0.1:51781, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-23T21:54:42,997 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:37831-0x1016afab2c80001, quorum=127.0.0.1:51781, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-23T21:54:42,997 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=37831 2024-11-23T21:54:42,998 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=37831 2024-11-23T21:54:42,998 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=37831 2024-11-23T21:54:42,998 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=37831 2024-11-23T21:54:42,998 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=37831 2024-11-23T21:54:43,013 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/adda16083778:0 server-side Connection retries=45 2024-11-23T21:54:43,013 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-23T21:54:43,013 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-23T21:54:43,013 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-23T21:54:43,013 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-23T21:54:43,013 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-23T21:54:43,013 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-23T21:54:43,013 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-23T21:54:43,014 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:45169 2024-11-23T21:54:43,015 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:45169 connecting to ZooKeeper ensemble=127.0.0.1:51781 2024-11-23T21:54:43,015 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-23T21:54:43,015 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-23T21:54:43,025 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:451690x0, quorum=127.0.0.1:51781, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-23T21:54:43,026 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:45169-0x1016afab2c80002 connected 2024-11-23T21:54:43,026 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:45169-0x1016afab2c80002, quorum=127.0.0.1:51781, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-23T21:54:43,026 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-23T21:54:43,027 INFO [Time-limited test {}] bucket.FileIOEngine(99): Allocating cache 400 MB, on the path:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d246ebad-447b-0ccb-984f-a43056e9385e/bucket.cache 2024-11-23T21:54:43,027 INFO [Time-limited test {}] bucket.BucketCache(364): Instantiating BucketCache with acceptableFactor: 0.95, minFactor: 0.85, extraFreeFactor: 0.1, singleFactor: 0.25, multiFactor: 0.5, memoryFactor: 0.25, useStrongRef: false 2024-11-23T21:54:43,028 INFO [Time-limited test {}] bucket.BucketCache(478): Starting BucketCachePersisterThread 2024-11-23T21:54:43,028 INFO [Time-limited test {}] bucket.BucketCachePersister(38): BucketCachePersister started with interval: 100 2024-11-23T21:54:43,028 INFO [Thread-119 {}] bucket.BucketCache(1530): Started retrieving bucket cache from file 2024-11-23T21:54:43,029 WARN [Thread-119 {}] bucket.BucketCache(1533): Persistence file missing! It's ok if it's first run after enabling persistent cache. 2024-11-23T21:54:43,029 INFO [Time-limited test {}] bucket.BucketCache(403): Started bucket cache; ioengine=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d246ebad-447b-0ccb-984f-a43056e9385e/bucket.cache, capacity=400 MB, blockSize=64 KB, writerThreadNum=3, writerQLen=64, persistencePath=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d246ebad-447b-0ccb-984f-a43056e9385e/bucket.persistence, bucketAllocator=org.apache.hadoop.hbase.io.hfile.bucket.BucketAllocator 2024-11-23T21:54:43,029 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-23T21:54:43,029 INFO [Thread-119 {}] bucket.BucketAllocator(343): Cache totalSize=418148352, buckets=199, bucket capacity=2101248=(4*525312)=(FEWEST_ITEMS_IN_BUCKET*(largest configured bucketcache size)) 2024-11-23T21:54:43,029 INFO [Thread-119 {}] bucket.BucketCache(413): Persistent bucket cache recovery from /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d246ebad-447b-0ccb-984f-a43056e9385e/bucket.persistence is complete. 2024-11-23T21:54:43,030 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:45169-0x1016afab2c80002, quorum=127.0.0.1:51781, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-23T21:54:43,032 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:45169-0x1016afab2c80002, quorum=127.0.0.1:51781, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-23T21:54:43,033 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=45169 2024-11-23T21:54:43,034 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=45169 2024-11-23T21:54:43,034 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=45169 2024-11-23T21:54:43,036 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=45169 2024-11-23T21:54:43,038 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=45169 2024-11-23T21:54:43,050 DEBUG [M:0;adda16083778:33953 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;adda16083778:33953 2024-11-23T21:54:43,050 INFO [master/adda16083778:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/adda16083778,33953,1732398882834 2024-11-23T21:54:43,059 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33953-0x1016afab2c80000, quorum=127.0.0.1:51781, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-23T21:54:43,059 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45169-0x1016afab2c80002, quorum=127.0.0.1:51781, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-23T21:54:43,059 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37831-0x1016afab2c80001, quorum=127.0.0.1:51781, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-23T21:54:43,059 DEBUG [master/adda16083778:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:33953-0x1016afab2c80000, quorum=127.0.0.1:51781, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/adda16083778,33953,1732398882834 2024-11-23T21:54:43,067 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33953-0x1016afab2c80000, quorum=127.0.0.1:51781, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-23T21:54:43,067 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37831-0x1016afab2c80001, quorum=127.0.0.1:51781, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-23T21:54:43,067 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45169-0x1016afab2c80002, quorum=127.0.0.1:51781, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-23T21:54:43,067 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37831-0x1016afab2c80001, quorum=127.0.0.1:51781, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-23T21:54:43,067 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45169-0x1016afab2c80002, quorum=127.0.0.1:51781, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-23T21:54:43,068 DEBUG [master/adda16083778:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:33953-0x1016afab2c80000, quorum=127.0.0.1:51781, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-11-23T21:54:43,068 INFO [master/adda16083778:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/adda16083778,33953,1732398882834 from backup master directory 2024-11-23T21:54:43,075 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45169-0x1016afab2c80002, quorum=127.0.0.1:51781, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-23T21:54:43,075 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37831-0x1016afab2c80001, quorum=127.0.0.1:51781, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-23T21:54:43,075 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33953-0x1016afab2c80000, quorum=127.0.0.1:51781, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/adda16083778,33953,1732398882834 2024-11-23T21:54:43,076 WARN [master/adda16083778:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-23T21:54:43,076 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33953-0x1016afab2c80000, quorum=127.0.0.1:51781, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-23T21:54:43,076 INFO [master/adda16083778:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=adda16083778,33953,1732398882834 2024-11-23T21:54:43,077 DEBUG [master/adda16083778:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/93be71cd-4dcf-2478-daf8-a2306a879141/e0862929-e242-795d-1e0b-fe7766981a5b/hbase.id] with ID: d343274d-4bd1-4b55-b460-a843638d8a30 2024-11-23T21:54:43,077 DEBUG [master/adda16083778:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/93be71cd-4dcf-2478-daf8-a2306a879141/e0862929-e242-795d-1e0b-fe7766981a5b/.tmp/hbase.id 2024-11-23T21:54:43,080 DEBUG [master/adda16083778:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/93be71cd-4dcf-2478-daf8-a2306a879141/e0862929-e242-795d-1e0b-fe7766981a5b/.tmp/hbase.id]:[file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/93be71cd-4dcf-2478-daf8-a2306a879141/e0862929-e242-795d-1e0b-fe7766981a5b/hbase.id] 2024-11-23T21:54:43,109 DEBUG [master/adda16083778:0:becomeActiveMaster {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-23T21:54:43,110 INFO [master/adda16083778:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem. 2024-11-23T21:54:43,112 INFO [master/adda16083778:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 2ms. 2024-11-23T21:54:43,117 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33953-0x1016afab2c80000, quorum=127.0.0.1:51781, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-23T21:54:43,117 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37831-0x1016afab2c80001, quorum=127.0.0.1:51781, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-23T21:54:43,117 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45169-0x1016afab2c80002, quorum=127.0.0.1:51781, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-23T21:54:43,121 INFO [master/adda16083778:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-23T21:54:43,123 INFO [master/adda16083778:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-11-23T21:54:43,123 INFO [master/adda16083778:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-23T21:54:43,142 INFO [master/adda16083778:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/93be71cd-4dcf-2478-daf8-a2306a879141/e0862929-e242-795d-1e0b-fe7766981a5b/MasterData/data/master/store 2024-11-23T21:54:43,148 DEBUG [master/adda16083778:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-23T21:54:43,149 DEBUG [master/adda16083778:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-23T21:54:43,149 INFO [master/adda16083778:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-23T21:54:43,149 DEBUG [master/adda16083778:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-23T21:54:43,149 DEBUG [master/adda16083778:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-23T21:54:43,149 DEBUG [master/adda16083778:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-23T21:54:43,149 INFO [master/adda16083778:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-23T21:54:43,149 DEBUG [master/adda16083778:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1732398883148Disabling compacts and flushes for region at 1732398883148Disabling writes for close at 1732398883149 (+1 ms)Writing region close event to WAL at 1732398883149Closed at 1732398883149 2024-11-23T21:54:43,149 WARN [master/adda16083778:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/93be71cd-4dcf-2478-daf8-a2306a879141/e0862929-e242-795d-1e0b-fe7766981a5b/MasterData/data/master/store/.initializing 2024-11-23T21:54:43,149 DEBUG [master/adda16083778:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/93be71cd-4dcf-2478-daf8-a2306a879141/e0862929-e242-795d-1e0b-fe7766981a5b/MasterData/WALs/adda16083778,33953,1732398882834 2024-11-23T21:54:43,154 INFO [master/adda16083778:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-23T21:54:43,157 INFO [master/adda16083778:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=adda16083778%2C33953%2C1732398882834, suffix=, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/93be71cd-4dcf-2478-daf8-a2306a879141/e0862929-e242-795d-1e0b-fe7766981a5b/MasterData/WALs/adda16083778,33953,1732398882834, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/93be71cd-4dcf-2478-daf8-a2306a879141/e0862929-e242-795d-1e0b-fe7766981a5b/MasterData/oldWALs, maxLogs=10 2024-11-23T21:54:43,173 INFO [master/adda16083778:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/93be71cd-4dcf-2478-daf8-a2306a879141/e0862929-e242-795d-1e0b-fe7766981a5b/MasterData/WALs/adda16083778,33953,1732398882834/adda16083778%2C33953%2C1732398882834.1732398883158 2024-11-23T21:54:43,174 DEBUG [master/adda16083778:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-11-23T21:54:43,174 DEBUG [master/adda16083778:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-11-23T21:54:43,174 DEBUG [master/adda16083778:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-23T21:54:43,174 DEBUG [master/adda16083778:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-11-23T21:54:43,174 DEBUG [master/adda16083778:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-11-23T21:54:43,175 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-11-23T21:54:43,179 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-11-23T21:54:43,179 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T21:54:43,180 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-23T21:54:43,180 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-11-23T21:54:43,184 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-11-23T21:54:43,184 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T21:54:43,184 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-23T21:54:43,184 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-11-23T21:54:43,187 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-11-23T21:54:43,187 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T21:54:43,188 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-23T21:54:43,188 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-11-23T21:54:43,192 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-11-23T21:54:43,192 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T21:54:43,192 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-23T21:54:43,192 DEBUG [master/adda16083778:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682 2024-11-23T21:54:43,193 DEBUG [master/adda16083778:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/93be71cd-4dcf-2478-daf8-a2306a879141/e0862929-e242-795d-1e0b-fe7766981a5b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-23T21:54:43,193 DEBUG [master/adda16083778:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/93be71cd-4dcf-2478-daf8-a2306a879141/e0862929-e242-795d-1e0b-fe7766981a5b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-23T21:54:43,193 DEBUG [master/adda16083778:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682 2024-11-23T21:54:43,194 DEBUG [master/adda16083778:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682 2024-11-23T21:54:43,194 DEBUG [master/adda16083778:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-23T21:54:43,194 DEBUG [master/adda16083778:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-11-23T21:54:43,200 DEBUG [master/adda16083778:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/93be71cd-4dcf-2478-daf8-a2306a879141/e0862929-e242-795d-1e0b-fe7766981a5b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-23T21:54:43,200 INFO [master/adda16083778:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=72121457, jitterRate=0.0746934562921524}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-23T21:54:43,201 DEBUG [master/adda16083778:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682: Writing region info on filesystem at 1732398883174Initializing all the Stores at 1732398883174Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732398883175 (+1 ms)Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732398883175Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732398883175Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732398883175Cleaning up temporary data from old regions at 1732398883194 (+19 ms)Region opened successfully at 1732398883201 (+7 ms) 2024-11-23T21:54:43,202 INFO [master/adda16083778:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-11-23T21:54:43,205 DEBUG [master/adda16083778:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@420a0abc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=adda16083778/172.17.0.2:0 2024-11-23T21:54:43,206 INFO [master/adda16083778:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating... 2024-11-23T21:54:43,206 INFO [master/adda16083778:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5 2024-11-23T21:54:43,206 INFO [master/adda16083778:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50 2024-11-23T21:54:43,206 INFO [master/adda16083778:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-23T21:54:43,207 INFO [master/adda16083778:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-23T21:54:43,207 INFO [master/adda16083778:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 0 msec 2024-11-23T21:54:43,207 INFO [master/adda16083778:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-11-23T21:54:43,209 INFO [master/adda16083778:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2024-11-23T21:54:43,210 DEBUG [master/adda16083778:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:33953-0x1016afab2c80000, quorum=127.0.0.1:51781, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error) 2024-11-23T21:54:43,241 DEBUG [master/adda16083778:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false 2024-11-23T21:54:43,241 INFO [master/adda16083778:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2024-11-23T21:54:43,243 DEBUG [master/adda16083778:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:33953-0x1016afab2c80000, quorum=127.0.0.1:51781, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error) 2024-11-23T21:54:43,250 DEBUG [master/adda16083778:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false 2024-11-23T21:54:43,251 INFO [master/adda16083778:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2024-11-23T21:54:43,252 DEBUG [master/adda16083778:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:33953-0x1016afab2c80000, quorum=127.0.0.1:51781, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error) 2024-11-23T21:54:43,258 DEBUG [master/adda16083778:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false 2024-11-23T21:54:43,260 DEBUG [master/adda16083778:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:33953-0x1016afab2c80000, quorum=127.0.0.1:51781, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error) 2024-11-23T21:54:43,267 DEBUG [master/adda16083778:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false 2024-11-23T21:54:43,269 DEBUG [master/adda16083778:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:33953-0x1016afab2c80000, quorum=127.0.0.1:51781, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error) 2024-11-23T21:54:43,275 DEBUG [master/adda16083778:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false 2024-11-23T21:54:43,284 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33953-0x1016afab2c80000, quorum=127.0.0.1:51781, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-23T21:54:43,284 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37831-0x1016afab2c80001, quorum=127.0.0.1:51781, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-23T21:54:43,284 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45169-0x1016afab2c80002, quorum=127.0.0.1:51781, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-23T21:54:43,284 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33953-0x1016afab2c80000, quorum=127.0.0.1:51781, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-23T21:54:43,284 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37831-0x1016afab2c80001, quorum=127.0.0.1:51781, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-23T21:54:43,284 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45169-0x1016afab2c80002, quorum=127.0.0.1:51781, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-23T21:54:43,285 INFO [master/adda16083778:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=adda16083778,33953,1732398882834, sessionid=0x1016afab2c80000, setting cluster-up flag (Was=false) 2024-11-23T21:54:43,300 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33953-0x1016afab2c80000, quorum=127.0.0.1:51781, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-23T21:54:43,301 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45169-0x1016afab2c80002, quorum=127.0.0.1:51781, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-23T21:54:43,301 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37831-0x1016afab2c80001, quorum=127.0.0.1:51781, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-23T21:54:43,325 DEBUG [master/adda16083778:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort 2024-11-23T21:54:43,327 DEBUG [master/adda16083778:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=adda16083778,33953,1732398882834 2024-11-23T21:54:43,342 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33953-0x1016afab2c80000, quorum=127.0.0.1:51781, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-23T21:54:43,342 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37831-0x1016afab2c80001, quorum=127.0.0.1:51781, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-23T21:54:43,342 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45169-0x1016afab2c80002, quorum=127.0.0.1:51781, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-23T21:54:43,367 DEBUG [master/adda16083778:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort 2024-11-23T21:54:43,369 DEBUG [master/adda16083778:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=adda16083778,33953,1732398882834 2024-11-23T21:54:43,369 INFO [master/adda16083778:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/93be71cd-4dcf-2478-daf8-a2306a879141/e0862929-e242-795d-1e0b-fe7766981a5b/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again 2024-11-23T21:54:43,371 DEBUG [master/adda16083778:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-23T21:54:43,371 INFO [master/adda16083778:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2 2024-11-23T21:54:43,371 INFO [master/adda16083778:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2024-11-23T21:54:43,372 DEBUG [master/adda16083778:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: adda16083778,33953,1732398882834 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2024-11-23T21:54:43,373 DEBUG [master/adda16083778:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/adda16083778:0, corePoolSize=5, maxPoolSize=5 2024-11-23T21:54:43,374 DEBUG [master/adda16083778:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/adda16083778:0, corePoolSize=5, maxPoolSize=5 2024-11-23T21:54:43,374 DEBUG [master/adda16083778:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/adda16083778:0, corePoolSize=5, maxPoolSize=5 2024-11-23T21:54:43,374 DEBUG [master/adda16083778:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/adda16083778:0, corePoolSize=5, maxPoolSize=5 2024-11-23T21:54:43,374 DEBUG [master/adda16083778:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/adda16083778:0, corePoolSize=10, maxPoolSize=10 2024-11-23T21:54:43,374 DEBUG [master/adda16083778:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/adda16083778:0, corePoolSize=1, maxPoolSize=1 2024-11-23T21:54:43,374 DEBUG [master/adda16083778:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/adda16083778:0, corePoolSize=2, maxPoolSize=2 2024-11-23T21:54:43,374 DEBUG [master/adda16083778:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/adda16083778:0, corePoolSize=1, maxPoolSize=1 2024-11-23T21:54:43,375 INFO [master/adda16083778:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732398913375 2024-11-23T21:54:43,375 INFO [master/adda16083778:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2024-11-23T21:54:43,375 INFO [master/adda16083778:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2024-11-23T21:54:43,376 INFO [master/adda16083778:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2024-11-23T21:54:43,376 INFO [master/adda16083778:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2024-11-23T21:54:43,376 INFO [master/adda16083778:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2024-11-23T21:54:43,376 INFO [master/adda16083778:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2024-11-23T21:54:43,376 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-23T21:54:43,376 INFO [master/adda16083778:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-23T21:54:43,376 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region 2024-11-23T21:54:43,376 INFO [master/adda16083778:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2024-11-23T21:54:43,377 INFO [master/adda16083778:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2024-11-23T21:54:43,377 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T21:54:43,377 INFO [master/adda16083778:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2024-11-23T21:54:43,377 INFO [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-11-23T21:54:43,377 INFO [master/adda16083778:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2024-11-23T21:54:43,377 INFO [master/adda16083778:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2024-11-23T21:54:43,378 DEBUG [master/adda16083778:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/adda16083778:0:becomeActiveMaster-HFileCleaner.large.0-1732398883377,5,FailOnTimeoutGroup] 2024-11-23T21:54:43,378 DEBUG [master/adda16083778:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/adda16083778:0:becomeActiveMaster-HFileCleaner.small.0-1732398883378,5,FailOnTimeoutGroup] 2024-11-23T21:54:43,378 INFO [master/adda16083778:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-23T21:54:43,378 INFO [master/adda16083778:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2024-11-23T21:54:43,379 INFO [master/adda16083778:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2024-11-23T21:54:43,379 INFO [master/adda16083778:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2024-11-23T21:54:43,393 INFO [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/93be71cd-4dcf-2478-daf8-a2306a879141/e0862929-e242-795d-1e0b-fe7766981a5b/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321 2024-11-23T21:54:43,393 INFO [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/93be71cd-4dcf-2478-daf8-a2306a879141/e0862929-e242-795d-1e0b-fe7766981a5b 2024-11-23T21:54:43,401 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-23T21:54:43,402 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family info of region 1588230740 2024-11-23T21:54:43,407 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-23T21:54:43,407 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T21:54:43,407 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-23T21:54:43,407 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family ns of region 1588230740 2024-11-23T21:54:43,410 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-23T21:54:43,410 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T21:54:43,410 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-23T21:54:43,410 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family rep_barrier of region 1588230740 2024-11-23T21:54:43,413 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-23T21:54:43,413 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T21:54:43,413 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-23T21:54:43,413 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family table of region 1588230740 2024-11-23T21:54:43,416 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-23T21:54:43,416 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T21:54:43,416 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-23T21:54:43,416 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-23T21:54:43,417 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/93be71cd-4dcf-2478-daf8-a2306a879141/e0862929-e242-795d-1e0b-fe7766981a5b/data/hbase/meta/1588230740 2024-11-23T21:54:43,417 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/93be71cd-4dcf-2478-daf8-a2306a879141/e0862929-e242-795d-1e0b-fe7766981a5b/data/hbase/meta/1588230740 2024-11-23T21:54:43,417 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-23T21:54:43,417 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-23T21:54:43,417 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-23T21:54:43,418 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-23T21:54:43,424 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/93be71cd-4dcf-2478-daf8-a2306a879141/e0862929-e242-795d-1e0b-fe7766981a5b/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-23T21:54:43,425 INFO [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=74938008, jitterRate=0.11666333675384521}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-23T21:54:43,426 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740: Writing region info on filesystem at 1732398883401Initializing all the Stores at 1732398883401Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732398883401Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732398883402 (+1 ms)Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732398883402Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732398883402Cleaning up temporary data from old regions at 1732398883417 (+15 ms)Region opened successfully at 1732398883425 (+8 ms) 2024-11-23T21:54:43,426 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-23T21:54:43,426 INFO [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-23T21:54:43,426 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-23T21:54:43,426 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-23T21:54:43,426 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-23T21:54:43,426 INFO [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-23T21:54:43,426 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1732398883426Disabling compacts and flushes for region at 1732398883426Disabling writes for close at 1732398883426Writing region close event to WAL at 1732398883426Closed at 1732398883426 2024-11-23T21:54:43,427 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-23T21:54:43,427 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta 2024-11-23T21:54:43,428 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-11-23T21:54:43,429 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-23T21:54:43,429 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2024-11-23T21:54:43,441 INFO [RS:0;adda16083778:37831 {}] regionserver.HRegionServer(746): ClusterId : d343274d-4bd1-4b55-b460-a843638d8a30 2024-11-23T21:54:43,441 INFO [RS:1;adda16083778:45169 {}] regionserver.HRegionServer(746): ClusterId : d343274d-4bd1-4b55-b460-a843638d8a30 2024-11-23T21:54:43,441 DEBUG [RS:1;adda16083778:45169 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-23T21:54:43,441 DEBUG [RS:0;adda16083778:37831 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-23T21:54:43,467 DEBUG [RS:1;adda16083778:45169 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-23T21:54:43,467 DEBUG [RS:0;adda16083778:37831 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-23T21:54:43,467 DEBUG [RS:1;adda16083778:45169 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-23T21:54:43,467 DEBUG [RS:0;adda16083778:37831 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-23T21:54:43,476 DEBUG [RS:0;adda16083778:37831 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-23T21:54:43,476 DEBUG [RS:1;adda16083778:45169 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-23T21:54:43,477 DEBUG [RS:0;adda16083778:37831 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@713cd310, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=adda16083778/172.17.0.2:0 2024-11-23T21:54:43,478 DEBUG [RS:1;adda16083778:45169 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@28eb34ba, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=adda16083778/172.17.0.2:0 2024-11-23T21:54:43,496 DEBUG [RS:1;adda16083778:45169 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:1;adda16083778:45169 2024-11-23T21:54:43,496 DEBUG [RS:0;adda16083778:37831 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;adda16083778:37831 2024-11-23T21:54:43,496 INFO [RS:1;adda16083778:45169 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-23T21:54:43,496 INFO [RS:0;adda16083778:37831 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-23T21:54:43,496 INFO [RS:1;adda16083778:45169 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-23T21:54:43,497 INFO [RS:0;adda16083778:37831 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-23T21:54:43,497 DEBUG [RS:1;adda16083778:45169 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-23T21:54:43,497 DEBUG [RS:0;adda16083778:37831 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-23T21:54:43,498 INFO [RS:0;adda16083778:37831 {}] regionserver.HRegionServer(2659): reportForDuty to master=adda16083778,33953,1732398882834 with port=37831, startcode=1732398882981 2024-11-23T21:54:43,498 INFO [RS:1;adda16083778:45169 {}] regionserver.HRegionServer(2659): reportForDuty to master=adda16083778,33953,1732398882834 with port=45169, startcode=1732398883012 2024-11-23T21:54:43,498 DEBUG [RS:0;adda16083778:37831 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-23T21:54:43,498 DEBUG [RS:1;adda16083778:45169 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-23T21:54:43,501 INFO [HMaster-EventLoopGroup-6-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60527, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=RegionServerStatusService 2024-11-23T21:54:43,501 INFO [HMaster-EventLoopGroup-6-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42963, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=RegionServerStatusService 2024-11-23T21:54:43,501 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=33953 {}] master.ServerManager(363): Checking decommissioned status of RegionServer adda16083778,37831,1732398882981 2024-11-23T21:54:43,502 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=33953 {}] master.ServerManager(517): Registering regionserver=adda16083778,37831,1732398882981 2024-11-23T21:54:43,503 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=33953 {}] master.ServerManager(363): Checking decommissioned status of RegionServer adda16083778,45169,1732398883012 2024-11-23T21:54:43,503 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=33953 {}] master.ServerManager(517): Registering regionserver=adda16083778,45169,1732398883012 2024-11-23T21:54:43,503 DEBUG [RS:0;adda16083778:37831 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/93be71cd-4dcf-2478-daf8-a2306a879141/e0862929-e242-795d-1e0b-fe7766981a5b 2024-11-23T21:54:43,503 DEBUG [RS:0;adda16083778:37831 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=file:/ 2024-11-23T21:54:43,504 DEBUG [RS:0;adda16083778:37831 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-23T21:54:43,505 DEBUG [RS:1;adda16083778:45169 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/93be71cd-4dcf-2478-daf8-a2306a879141/e0862929-e242-795d-1e0b-fe7766981a5b 2024-11-23T21:54:43,505 DEBUG [RS:1;adda16083778:45169 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=file:/ 2024-11-23T21:54:43,505 DEBUG [RS:1;adda16083778:45169 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-23T21:54:43,543 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33953-0x1016afab2c80000, quorum=127.0.0.1:51781, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-23T21:54:43,544 DEBUG [RS:0;adda16083778:37831 {}] zookeeper.ZKUtil(111): regionserver:37831-0x1016afab2c80001, quorum=127.0.0.1:51781, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/adda16083778,37831,1732398882981 2024-11-23T21:54:43,544 DEBUG [RS:1;adda16083778:45169 {}] zookeeper.ZKUtil(111): regionserver:45169-0x1016afab2c80002, quorum=127.0.0.1:51781, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/adda16083778,45169,1732398883012 2024-11-23T21:54:43,544 WARN [RS:0;adda16083778:37831 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-23T21:54:43,544 WARN [RS:1;adda16083778:45169 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-23T21:54:43,544 INFO [RS:0;adda16083778:37831 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-23T21:54:43,544 INFO [RS:1;adda16083778:45169 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-23T21:54:43,544 DEBUG [RS:0;adda16083778:37831 {}] regionserver.HRegionServer(1793): logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/93be71cd-4dcf-2478-daf8-a2306a879141/e0862929-e242-795d-1e0b-fe7766981a5b/WALs/adda16083778,37831,1732398882981 2024-11-23T21:54:43,544 DEBUG [RS:1;adda16083778:45169 {}] regionserver.HRegionServer(1793): logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/93be71cd-4dcf-2478-daf8-a2306a879141/e0862929-e242-795d-1e0b-fe7766981a5b/WALs/adda16083778,45169,1732398883012 2024-11-23T21:54:43,545 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [adda16083778,45169,1732398883012] 2024-11-23T21:54:43,545 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [adda16083778,37831,1732398882981] 2024-11-23T21:54:43,551 INFO [RS:1;adda16083778:45169 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-23T21:54:43,551 INFO [RS:0;adda16083778:37831 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-23T21:54:43,555 INFO [RS:1;adda16083778:45169 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-23T21:54:43,556 INFO [RS:1;adda16083778:45169 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-23T21:54:43,556 INFO [RS:1;adda16083778:45169 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-23T21:54:43,556 INFO [RS:1;adda16083778:45169 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-23T21:54:43,556 INFO [RS:1;adda16083778:45169 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-23T21:54:43,557 INFO [RS:1;adda16083778:45169 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-23T21:54:43,557 DEBUG [RS:1;adda16083778:45169 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/adda16083778:0, corePoolSize=1, maxPoolSize=1 2024-11-23T21:54:43,557 DEBUG [RS:1;adda16083778:45169 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/adda16083778:0, corePoolSize=1, maxPoolSize=1 2024-11-23T21:54:43,557 DEBUG [RS:1;adda16083778:45169 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/adda16083778:0, corePoolSize=1, maxPoolSize=1 2024-11-23T21:54:43,557 DEBUG [RS:1;adda16083778:45169 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/adda16083778:0, corePoolSize=1, maxPoolSize=1 2024-11-23T21:54:43,557 DEBUG [RS:1;adda16083778:45169 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/adda16083778:0, corePoolSize=1, maxPoolSize=1 2024-11-23T21:54:43,557 INFO [RS:0;adda16083778:37831 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-23T21:54:43,557 DEBUG [RS:1;adda16083778:45169 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/adda16083778:0, corePoolSize=2, maxPoolSize=2 2024-11-23T21:54:43,558 DEBUG [RS:1;adda16083778:45169 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/adda16083778:0, corePoolSize=1, maxPoolSize=1 2024-11-23T21:54:43,558 DEBUG [RS:1;adda16083778:45169 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/adda16083778:0, corePoolSize=1, maxPoolSize=1 2024-11-23T21:54:43,558 DEBUG [RS:1;adda16083778:45169 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/adda16083778:0, corePoolSize=1, maxPoolSize=1 2024-11-23T21:54:43,558 INFO [RS:0;adda16083778:37831 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-23T21:54:43,558 DEBUG [RS:1;adda16083778:45169 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/adda16083778:0, corePoolSize=1, maxPoolSize=1 2024-11-23T21:54:43,558 INFO [RS:0;adda16083778:37831 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-23T21:54:43,558 DEBUG [RS:1;adda16083778:45169 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/adda16083778:0, corePoolSize=1, maxPoolSize=1 2024-11-23T21:54:43,558 DEBUG [RS:1;adda16083778:45169 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/adda16083778:0, corePoolSize=1, maxPoolSize=1 2024-11-23T21:54:43,558 INFO [RS:0;adda16083778:37831 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-23T21:54:43,558 DEBUG [RS:1;adda16083778:45169 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/adda16083778:0, corePoolSize=3, maxPoolSize=3 2024-11-23T21:54:43,558 DEBUG [RS:1;adda16083778:45169 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/adda16083778:0, corePoolSize=3, maxPoolSize=3 2024-11-23T21:54:43,559 INFO [RS:0;adda16083778:37831 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-23T21:54:43,559 INFO [RS:0;adda16083778:37831 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-23T21:54:43,559 DEBUG [RS:0;adda16083778:37831 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/adda16083778:0, corePoolSize=1, maxPoolSize=1 2024-11-23T21:54:43,559 DEBUG [RS:0;adda16083778:37831 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/adda16083778:0, corePoolSize=1, maxPoolSize=1 2024-11-23T21:54:43,559 DEBUG [RS:0;adda16083778:37831 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/adda16083778:0, corePoolSize=1, maxPoolSize=1 2024-11-23T21:54:43,559 DEBUG [RS:0;adda16083778:37831 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/adda16083778:0, corePoolSize=1, maxPoolSize=1 2024-11-23T21:54:43,560 DEBUG [RS:0;adda16083778:37831 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/adda16083778:0, corePoolSize=1, maxPoolSize=1 2024-11-23T21:54:43,560 DEBUG [RS:0;adda16083778:37831 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/adda16083778:0, corePoolSize=2, maxPoolSize=2 2024-11-23T21:54:43,560 DEBUG [RS:0;adda16083778:37831 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/adda16083778:0, corePoolSize=1, maxPoolSize=1 2024-11-23T21:54:43,560 DEBUG [RS:0;adda16083778:37831 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/adda16083778:0, corePoolSize=1, maxPoolSize=1 2024-11-23T21:54:43,560 DEBUG [RS:0;adda16083778:37831 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/adda16083778:0, corePoolSize=1, maxPoolSize=1 2024-11-23T21:54:43,560 DEBUG [RS:0;adda16083778:37831 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/adda16083778:0, corePoolSize=1, maxPoolSize=1 2024-11-23T21:54:43,560 DEBUG [RS:0;adda16083778:37831 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/adda16083778:0, corePoolSize=1, maxPoolSize=1 2024-11-23T21:54:43,560 DEBUG [RS:0;adda16083778:37831 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/adda16083778:0, corePoolSize=1, maxPoolSize=1 2024-11-23T21:54:43,561 DEBUG [RS:0;adda16083778:37831 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/adda16083778:0, corePoolSize=3, maxPoolSize=3 2024-11-23T21:54:43,561 DEBUG [RS:0;adda16083778:37831 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/adda16083778:0, corePoolSize=3, maxPoolSize=3 2024-11-23T21:54:43,561 INFO [RS:1;adda16083778:45169 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-23T21:54:43,561 INFO [RS:1;adda16083778:45169 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-23T21:54:43,561 INFO [RS:1;adda16083778:45169 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-23T21:54:43,561 INFO [RS:0;adda16083778:37831 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-23T21:54:43,561 INFO [RS:1;adda16083778:45169 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-23T21:54:43,561 INFO [RS:1;adda16083778:45169 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-23T21:54:43,561 INFO [RS:0;adda16083778:37831 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-23T21:54:43,561 INFO [RS:1;adda16083778:45169 {}] hbase.ChoreService(168): Chore ScheduledChore name=adda16083778,45169,1732398883012-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-23T21:54:43,562 INFO [RS:0;adda16083778:37831 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-23T21:54:43,562 INFO [RS:0;adda16083778:37831 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-23T21:54:43,562 INFO [RS:0;adda16083778:37831 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-23T21:54:43,562 INFO [RS:0;adda16083778:37831 {}] hbase.ChoreService(168): Chore ScheduledChore name=adda16083778,37831,1732398882981-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-23T21:54:43,577 INFO [RS:0;adda16083778:37831 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-23T21:54:43,577 INFO [RS:0;adda16083778:37831 {}] hbase.ChoreService(168): Chore ScheduledChore name=adda16083778,37831,1732398882981-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-23T21:54:43,577 INFO [RS:1;adda16083778:45169 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-23T21:54:43,577 INFO [RS:0;adda16083778:37831 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-23T21:54:43,577 INFO [RS:1;adda16083778:45169 {}] hbase.ChoreService(168): Chore ScheduledChore name=adda16083778,45169,1732398883012-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-23T21:54:43,577 INFO [RS:0;adda16083778:37831 {}] regionserver.Replication(171): adda16083778,37831,1732398882981 started 2024-11-23T21:54:43,577 INFO [RS:1;adda16083778:45169 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-23T21:54:43,577 INFO [RS:1;adda16083778:45169 {}] regionserver.Replication(171): adda16083778,45169,1732398883012 started 2024-11-23T21:54:43,579 WARN [adda16083778:33953 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-23T21:54:43,591 INFO [RS:0;adda16083778:37831 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-23T21:54:43,591 INFO [RS:0;adda16083778:37831 {}] regionserver.HRegionServer(1482): Serving as adda16083778,37831,1732398882981, RpcServer on adda16083778/172.17.0.2:37831, sessionid=0x1016afab2c80001 2024-11-23T21:54:43,591 DEBUG [RS:0;adda16083778:37831 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-23T21:54:43,591 DEBUG [RS:0;adda16083778:37831 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager adda16083778,37831,1732398882981 2024-11-23T21:54:43,591 DEBUG [RS:0;adda16083778:37831 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'adda16083778,37831,1732398882981' 2024-11-23T21:54:43,591 DEBUG [RS:0;adda16083778:37831 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-23T21:54:43,592 DEBUG [RS:0;adda16083778:37831 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-23T21:54:43,592 INFO [RS:1;adda16083778:45169 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-23T21:54:43,592 INFO [RS:1;adda16083778:45169 {}] regionserver.HRegionServer(1482): Serving as adda16083778,45169,1732398883012, RpcServer on adda16083778/172.17.0.2:45169, sessionid=0x1016afab2c80002 2024-11-23T21:54:43,592 DEBUG [RS:0;adda16083778:37831 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-23T21:54:43,592 DEBUG [RS:1;adda16083778:45169 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-23T21:54:43,592 DEBUG [RS:0;adda16083778:37831 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-23T21:54:43,592 DEBUG [RS:1;adda16083778:45169 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager adda16083778,45169,1732398883012 2024-11-23T21:54:43,592 DEBUG [RS:0;adda16083778:37831 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager adda16083778,37831,1732398882981 2024-11-23T21:54:43,592 DEBUG [RS:1;adda16083778:45169 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'adda16083778,45169,1732398883012' 2024-11-23T21:54:43,592 DEBUG [RS:0;adda16083778:37831 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'adda16083778,37831,1732398882981' 2024-11-23T21:54:43,592 DEBUG [RS:1;adda16083778:45169 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-23T21:54:43,592 DEBUG [RS:0;adda16083778:37831 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-23T21:54:43,593 DEBUG [RS:0;adda16083778:37831 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-23T21:54:43,593 DEBUG [RS:1;adda16083778:45169 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-23T21:54:43,594 DEBUG [RS:1;adda16083778:45169 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-23T21:54:43,594 DEBUG [RS:0;adda16083778:37831 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-23T21:54:43,594 INFO [RS:0;adda16083778:37831 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-23T21:54:43,594 DEBUG [RS:1;adda16083778:45169 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-23T21:54:43,594 DEBUG [RS:1;adda16083778:45169 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager adda16083778,45169,1732398883012 2024-11-23T21:54:43,594 INFO [RS:0;adda16083778:37831 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-23T21:54:43,594 DEBUG [RS:1;adda16083778:45169 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'adda16083778,45169,1732398883012' 2024-11-23T21:54:43,594 DEBUG [RS:1;adda16083778:45169 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-23T21:54:43,594 DEBUG [RS:1;adda16083778:45169 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-23T21:54:43,595 DEBUG [RS:1;adda16083778:45169 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-23T21:54:43,595 INFO [RS:1;adda16083778:45169 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-23T21:54:43,595 INFO [RS:1;adda16083778:45169 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-23T21:54:43,695 INFO [RS:0;adda16083778:37831 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-23T21:54:43,696 INFO [RS:1;adda16083778:45169 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-23T21:54:43,697 INFO [RS:1;adda16083778:45169 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=adda16083778%2C45169%2C1732398883012, suffix=, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/93be71cd-4dcf-2478-daf8-a2306a879141/e0862929-e242-795d-1e0b-fe7766981a5b/WALs/adda16083778,45169,1732398883012, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/93be71cd-4dcf-2478-daf8-a2306a879141/e0862929-e242-795d-1e0b-fe7766981a5b/oldWALs, maxLogs=55 2024-11-23T21:54:43,697 INFO [RS:0;adda16083778:37831 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=adda16083778%2C37831%2C1732398882981, suffix=, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/93be71cd-4dcf-2478-daf8-a2306a879141/e0862929-e242-795d-1e0b-fe7766981a5b/WALs/adda16083778,37831,1732398882981, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/93be71cd-4dcf-2478-daf8-a2306a879141/e0862929-e242-795d-1e0b-fe7766981a5b/oldWALs, maxLogs=55 2024-11-23T21:54:43,719 INFO [RS:0;adda16083778:37831 {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/93be71cd-4dcf-2478-daf8-a2306a879141/e0862929-e242-795d-1e0b-fe7766981a5b/WALs/adda16083778,37831,1732398882981/adda16083778%2C37831%2C1732398882981.1732398883703 2024-11-23T21:54:43,719 INFO [RS:1;adda16083778:45169 {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/93be71cd-4dcf-2478-daf8-a2306a879141/e0862929-e242-795d-1e0b-fe7766981a5b/WALs/adda16083778,45169,1732398883012/adda16083778%2C45169%2C1732398883012.1732398883701 2024-11-23T21:54:43,719 DEBUG [RS:1;adda16083778:45169 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-11-23T21:54:43,719 DEBUG [RS:0;adda16083778:37831 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-11-23T21:54:43,830 DEBUG [adda16083778:33953 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=2, allServersCount=2 2024-11-23T21:54:43,831 DEBUG [adda16083778:33953 {}] balancer.BalancerClusterState(204): Hosts are {adda16083778=0} racks are {/default-rack=0} 2024-11-23T21:54:43,832 DEBUG [adda16083778:33953 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-11-23T21:54:43,832 DEBUG [adda16083778:33953 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-11-23T21:54:43,833 DEBUG [adda16083778:33953 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-11-23T21:54:43,833 DEBUG [adda16083778:33953 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-11-23T21:54:43,833 INFO [adda16083778:33953 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-11-23T21:54:43,833 INFO [adda16083778:33953 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-11-23T21:54:43,833 DEBUG [adda16083778:33953 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-11-23T21:54:43,834 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=adda16083778,37831,1732398882981 2024-11-23T21:54:43,835 INFO [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as adda16083778,37831,1732398882981, state=OPENING 2024-11-23T21:54:43,866 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2024-11-23T21:54:43,875 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37831-0x1016afab2c80001, quorum=127.0.0.1:51781, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-23T21:54:43,875 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45169-0x1016afab2c80002, quorum=127.0.0.1:51781, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-23T21:54:43,876 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33953-0x1016afab2c80000, quorum=127.0.0.1:51781, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-23T21:54:43,877 DEBUG [PEWorker-3 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-23T21:54:43,877 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-23T21:54:43,877 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-23T21:54:43,877 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=adda16083778,37831,1732398882981}] 2024-11-23T21:54:43,877 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-23T21:54:44,032 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-23T21:54:44,036 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-7-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47083, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-23T21:54:44,039 INFO [RS_OPEN_META-regionserver/adda16083778:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-11-23T21:54:44,039 INFO [RS_OPEN_META-regionserver/adda16083778:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-23T21:54:44,040 INFO [RS_OPEN_META-regionserver/adda16083778:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-11-23T21:54:44,041 INFO [RS_OPEN_META-regionserver/adda16083778:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=adda16083778%2C37831%2C1732398882981.meta, suffix=.meta, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/93be71cd-4dcf-2478-daf8-a2306a879141/e0862929-e242-795d-1e0b-fe7766981a5b/WALs/adda16083778,37831,1732398882981, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/93be71cd-4dcf-2478-daf8-a2306a879141/e0862929-e242-795d-1e0b-fe7766981a5b/oldWALs, maxLogs=55 2024-11-23T21:54:44,058 INFO [RS_OPEN_META-regionserver/adda16083778:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/93be71cd-4dcf-2478-daf8-a2306a879141/e0862929-e242-795d-1e0b-fe7766981a5b/WALs/adda16083778,37831,1732398882981/adda16083778%2C37831%2C1732398882981.meta.1732398884042.meta 2024-11-23T21:54:44,058 DEBUG [RS_OPEN_META-regionserver/adda16083778:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-11-23T21:54:44,059 DEBUG [RS_OPEN_META-regionserver/adda16083778:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-11-23T21:54:44,059 DEBUG [RS_OPEN_META-regionserver/adda16083778:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-11-23T21:54:44,059 DEBUG [RS_OPEN_META-regionserver/adda16083778:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-11-23T21:54:44,059 INFO [RS_OPEN_META-regionserver/adda16083778:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-11-23T21:54:44,059 DEBUG [RS_OPEN_META-regionserver/adda16083778:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-11-23T21:54:44,059 DEBUG [RS_OPEN_META-regionserver/adda16083778:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-23T21:54:44,060 DEBUG [RS_OPEN_META-regionserver/adda16083778:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-11-23T21:54:44,060 DEBUG [RS_OPEN_META-regionserver/adda16083778:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-11-23T21:54:44,060 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family info of region 1588230740 2024-11-23T21:54:44,061 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-23T21:54:44,061 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T21:54:44,062 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-23T21:54:44,062 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family ns of region 1588230740 2024-11-23T21:54:44,062 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-23T21:54:44,063 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T21:54:44,063 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-23T21:54:44,063 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family rep_barrier of region 1588230740 2024-11-23T21:54:44,063 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-23T21:54:44,063 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T21:54:44,063 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-23T21:54:44,063 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family table of region 1588230740 2024-11-23T21:54:44,064 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-23T21:54:44,064 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T21:54:44,064 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-23T21:54:44,064 DEBUG [RS_OPEN_META-regionserver/adda16083778:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-23T21:54:44,065 DEBUG [RS_OPEN_META-regionserver/adda16083778:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/93be71cd-4dcf-2478-daf8-a2306a879141/e0862929-e242-795d-1e0b-fe7766981a5b/data/hbase/meta/1588230740 2024-11-23T21:54:44,065 DEBUG [RS_OPEN_META-regionserver/adda16083778:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/93be71cd-4dcf-2478-daf8-a2306a879141/e0862929-e242-795d-1e0b-fe7766981a5b/data/hbase/meta/1588230740 2024-11-23T21:54:44,065 DEBUG [RS_OPEN_META-regionserver/adda16083778:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-23T21:54:44,065 DEBUG [RS_OPEN_META-regionserver/adda16083778:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-23T21:54:44,065 DEBUG [RS_OPEN_META-regionserver/adda16083778:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-23T21:54:44,066 DEBUG [RS_OPEN_META-regionserver/adda16083778:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-23T21:54:44,066 INFO [RS_OPEN_META-regionserver/adda16083778:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=70016465, jitterRate=0.04332663118839264}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-23T21:54:44,066 DEBUG [RS_OPEN_META-regionserver/adda16083778:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-11-23T21:54:44,067 DEBUG [RS_OPEN_META-regionserver/adda16083778:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1732398884060Writing region info on filesystem at 1732398884060Initializing all the Stores at 1732398884060Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732398884060Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732398884060Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732398884060Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732398884060Cleaning up temporary data from old regions at 1732398884065 (+5 ms)Running coprocessor post-open hooks at 1732398884067 (+2 ms)Region opened successfully at 1732398884067 2024-11-23T21:54:44,068 INFO [RS_OPEN_META-regionserver/adda16083778:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1732398884031 2024-11-23T21:54:44,071 DEBUG [RS_OPEN_META-regionserver/adda16083778:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-11-23T21:54:44,071 INFO [RS_OPEN_META-regionserver/adda16083778:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-11-23T21:54:44,071 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=adda16083778,37831,1732398882981 2024-11-23T21:54:44,072 INFO [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as adda16083778,37831,1732398882981, state=OPEN 2024-11-23T21:54:44,099 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33953-0x1016afab2c80000, quorum=127.0.0.1:51781, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-23T21:54:44,099 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37831-0x1016afab2c80001, quorum=127.0.0.1:51781, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-23T21:54:44,099 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45169-0x1016afab2c80002, quorum=127.0.0.1:51781, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-23T21:54:44,099 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=adda16083778,37831,1732398882981 2024-11-23T21:54:44,099 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-23T21:54:44,099 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-23T21:54:44,099 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-23T21:54:44,103 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2 2024-11-23T21:54:44,103 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=adda16083778,37831,1732398882981 in 222 msec 2024-11-23T21:54:44,107 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1 2024-11-23T21:54:44,107 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 676 msec 2024-11-23T21:54:44,108 DEBUG [PEWorker-2 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-23T21:54:44,108 INFO [PEWorker-2 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces 2024-11-23T21:54:44,112 DEBUG [PEWorker-2 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-23T21:54:44,112 DEBUG [PEWorker-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=adda16083778,37831,1732398882981, seqNum=-1] 2024-11-23T21:54:44,112 DEBUG [PEWorker-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-23T21:54:44,114 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-7-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41113, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-23T21:54:44,119 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 748 msec 2024-11-23T21:54:44,119 INFO [master/adda16083778:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1732398884119, completionTime=-1 2024-11-23T21:54:44,120 INFO [master/adda16083778:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=2; waited=0ms, expected min=2 server(s), max=2 server(s), master is running 2024-11-23T21:54:44,120 DEBUG [master/adda16083778:0:becomeActiveMaster {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-23T21:54:44,123 INFO [master/adda16083778:0:becomeActiveMaster {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-11-23T21:54:44,123 INFO [master/adda16083778:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732398944123 2024-11-23T21:54:44,123 INFO [master/adda16083778:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732399004123 2024-11-23T21:54:44,123 INFO [master/adda16083778:0:becomeActiveMaster {}] assignment.AssignmentManager(1783): Joined the cluster in 3 msec 2024-11-23T21:54:44,123 INFO [master/adda16083778:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=adda16083778,33953,1732398882834-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-23T21:54:44,124 INFO [master/adda16083778:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=adda16083778,33953,1732398882834-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-23T21:54:44,124 INFO [master/adda16083778:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=adda16083778,33953,1732398882834-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-23T21:54:44,124 INFO [master/adda16083778:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-adda16083778:33953, period=300000, unit=MILLISECONDS is enabled. 2024-11-23T21:54:44,124 INFO [master/adda16083778:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2024-11-23T21:54:44,124 INFO [master/adda16083778:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled. 2024-11-23T21:54:44,127 DEBUG [master/adda16083778:0.Chore.1 {}] janitor.CatalogJanitor(180): 2024-11-23T21:54:44,130 INFO [master/adda16083778:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 1.054sec 2024-11-23T21:54:44,130 INFO [master/adda16083778:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2024-11-23T21:54:44,130 INFO [master/adda16083778:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2024-11-23T21:54:44,130 INFO [master/adda16083778:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2024-11-23T21:54:44,130 INFO [master/adda16083778:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2024-11-23T21:54:44,131 INFO [master/adda16083778:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2024-11-23T21:54:44,131 INFO [master/adda16083778:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=adda16083778,33953,1732398882834-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-23T21:54:44,131 INFO [master/adda16083778:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=adda16083778,33953,1732398882834-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2024-11-23T21:54:44,132 DEBUG [master/adda16083778:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds 2024-11-23T21:54:44,132 INFO [master/adda16083778:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2024-11-23T21:54:44,132 INFO [master/adda16083778:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=adda16083778,33953,1732398882834-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-23T21:54:44,141 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@689405c2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T21:54:44,141 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request adda16083778,33953,-1 for getting cluster id 2024-11-23T21:54:44,141 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-23T21:54:44,142 DEBUG [HMaster-EventLoopGroup-6-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd343274d-4bd1-4b55-b460-a843638d8a30' 2024-11-23T21:54:44,143 DEBUG [RPCClient-NioEventLoopGroup-5-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-23T21:54:44,143 DEBUG [RPCClient-NioEventLoopGroup-5-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d343274d-4bd1-4b55-b460-a843638d8a30" 2024-11-23T21:54:44,143 DEBUG [RPCClient-NioEventLoopGroup-5-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@59c195f0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T21:54:44,143 DEBUG [RPCClient-NioEventLoopGroup-5-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [adda16083778,33953,-1] 2024-11-23T21:54:44,143 DEBUG [RPCClient-NioEventLoopGroup-5-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-23T21:54:44,144 DEBUG [RPCClient-NioEventLoopGroup-5-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T21:54:44,145 INFO [HMaster-EventLoopGroup-6-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42858, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-23T21:54:44,147 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1d9d15ba, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T21:54:44,147 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-23T21:54:44,149 DEBUG [RPCClient-NioEventLoopGroup-5-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=adda16083778,37831,1732398882981, seqNum=-1] 2024-11-23T21:54:44,149 DEBUG [RPCClient-NioEventLoopGroup-5-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-23T21:54:44,151 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-7-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34746, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-23T21:54:44,153 INFO [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=adda16083778,33953,1732398882834 2024-11-23T21:54:44,154 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-23T21:54:44,155 DEBUG [RPCClient-NioEventLoopGroup-5-6 {}] client.AsyncConnectionImpl(321): The fetched master address is adda16083778,33953,1732398882834 2024-11-23T21:54:44,155 DEBUG [RPCClient-NioEventLoopGroup-5-6 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@43b0e51c 2024-11-23T21:54:44,155 DEBUG [RPCClient-NioEventLoopGroup-5-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-23T21:54:44,157 INFO [HMaster-EventLoopGroup-6-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42862, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-23T21:54:44,158 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'testBlockEvictionOnGracefulStop1732398884153', {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-23T21:54:44,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=testBlockEvictionOnGracefulStop1732398884153 2024-11-23T21:54:44,160 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=testBlockEvictionOnGracefulStop1732398884153 execute state=CREATE_TABLE_PRE_OPERATION 2024-11-23T21:54:44,161 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T21:54:44,161 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "testBlockEvictionOnGracefulStop1732398884153" procId is: 4 2024-11-23T21:54:44,162 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=testBlockEvictionOnGracefulStop1732398884153 execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-23T21:54:44,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-23T21:54:44,172 INFO [RegionOpenAndInit-testBlockEvictionOnGracefulStop1732398884153-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => d1d01dcb0a94d6c09a6bd0b1d689b7ea, NAME => 'testBlockEvictionOnGracefulStop1732398884153,,1732398884158.d1d01dcb0a94d6c09a6bd0b1d689b7ea.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testBlockEvictionOnGracefulStop1732398884153', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/93be71cd-4dcf-2478-daf8-a2306a879141/e0862929-e242-795d-1e0b-fe7766981a5b 2024-11-23T21:54:44,178 DEBUG [RegionOpenAndInit-testBlockEvictionOnGracefulStop1732398884153-pool-0 {}] regionserver.HRegion(898): Instantiated testBlockEvictionOnGracefulStop1732398884153,,1732398884158.d1d01dcb0a94d6c09a6bd0b1d689b7ea.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-23T21:54:44,178 DEBUG [RegionOpenAndInit-testBlockEvictionOnGracefulStop1732398884153-pool-0 {}] regionserver.HRegion(1722): Closing d1d01dcb0a94d6c09a6bd0b1d689b7ea, disabling compactions & flushes 2024-11-23T21:54:44,178 INFO [RegionOpenAndInit-testBlockEvictionOnGracefulStop1732398884153-pool-0 {}] regionserver.HRegion(1755): Closing region testBlockEvictionOnGracefulStop1732398884153,,1732398884158.d1d01dcb0a94d6c09a6bd0b1d689b7ea. 2024-11-23T21:54:44,178 DEBUG [RegionOpenAndInit-testBlockEvictionOnGracefulStop1732398884153-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on testBlockEvictionOnGracefulStop1732398884153,,1732398884158.d1d01dcb0a94d6c09a6bd0b1d689b7ea. 2024-11-23T21:54:44,178 DEBUG [RegionOpenAndInit-testBlockEvictionOnGracefulStop1732398884153-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on testBlockEvictionOnGracefulStop1732398884153,,1732398884158.d1d01dcb0a94d6c09a6bd0b1d689b7ea. after waiting 0 ms 2024-11-23T21:54:44,178 DEBUG [RegionOpenAndInit-testBlockEvictionOnGracefulStop1732398884153-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region testBlockEvictionOnGracefulStop1732398884153,,1732398884158.d1d01dcb0a94d6c09a6bd0b1d689b7ea. 2024-11-23T21:54:44,178 INFO [RegionOpenAndInit-testBlockEvictionOnGracefulStop1732398884153-pool-0 {}] regionserver.HRegion(1973): Closed testBlockEvictionOnGracefulStop1732398884153,,1732398884158.d1d01dcb0a94d6c09a6bd0b1d689b7ea. 2024-11-23T21:54:44,178 DEBUG [RegionOpenAndInit-testBlockEvictionOnGracefulStop1732398884153-pool-0 {}] regionserver.HRegion(1676): Region close journal for d1d01dcb0a94d6c09a6bd0b1d689b7ea: Waiting for close lock at 1732398884178Disabling compacts and flushes for region at 1732398884178Disabling writes for close at 1732398884178Writing region close event to WAL at 1732398884178Closed at 1732398884178 2024-11-23T21:54:44,179 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=testBlockEvictionOnGracefulStop1732398884153 execute state=CREATE_TABLE_ADD_TO_META 2024-11-23T21:54:44,180 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testBlockEvictionOnGracefulStop1732398884153,,1732398884158.d1d01dcb0a94d6c09a6bd0b1d689b7ea.","families":{"info":[{"qualifier":"regioninfo","vlen":78,"tag":[],"timestamp":"1732398884180"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732398884180"}]},"ts":"1732398884180"} 2024-11-23T21:54:44,183 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-23T21:54:44,184 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=testBlockEvictionOnGracefulStop1732398884153 execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-23T21:54:44,184 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testBlockEvictionOnGracefulStop1732398884153","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732398884184"}]},"ts":"1732398884184"} 2024-11-23T21:54:44,186 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=testBlockEvictionOnGracefulStop1732398884153, state=ENABLING in hbase:meta 2024-11-23T21:54:44,187 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(204): Hosts are {adda16083778=0} racks are {/default-rack=0} 2024-11-23T21:54:44,187 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-11-23T21:54:44,187 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-11-23T21:54:44,187 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-11-23T21:54:44,187 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-11-23T21:54:44,187 INFO [PEWorker-3 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-11-23T21:54:44,187 INFO [PEWorker-3 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-11-23T21:54:44,187 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-11-23T21:54:44,188 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1732398884153, region=d1d01dcb0a94d6c09a6bd0b1d689b7ea, ASSIGN}] 2024-11-23T21:54:44,189 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1732398884153, region=d1d01dcb0a94d6c09a6bd0b1d689b7ea, ASSIGN 2024-11-23T21:54:44,190 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1732398884153, region=d1d01dcb0a94d6c09a6bd0b1d689b7ea, ASSIGN; state=OFFLINE, location=adda16083778,37831,1732398882981; forceNewPlan=false, retain=false 2024-11-23T21:54:44,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-23T21:54:44,341 INFO [adda16083778:33953 {}] balancer.BaseLoadBalancer(388): Reassigned 1 regions. 1 retained the pre-restart assignment. 2024-11-23T21:54:44,342 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=d1d01dcb0a94d6c09a6bd0b1d689b7ea, regionState=OPENING, regionLocation=adda16083778,37831,1732398882981 2024-11-23T21:54:44,348 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-8-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1732398884153, region=d1d01dcb0a94d6c09a6bd0b1d689b7ea, ASSIGN because future has completed 2024-11-23T21:54:44,349 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure d1d01dcb0a94d6c09a6bd0b1d689b7ea, server=adda16083778,37831,1732398882981}] 2024-11-23T21:54:44,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-23T21:54:44,505 INFO [RS_OPEN_REGION-regionserver/adda16083778:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(132): Open testBlockEvictionOnGracefulStop1732398884153,,1732398884158.d1d01dcb0a94d6c09a6bd0b1d689b7ea. 2024-11-23T21:54:44,505 DEBUG [RS_OPEN_REGION-regionserver/adda16083778:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7752): Opening region: {ENCODED => d1d01dcb0a94d6c09a6bd0b1d689b7ea, NAME => 'testBlockEvictionOnGracefulStop1732398884153,,1732398884158.d1d01dcb0a94d6c09a6bd0b1d689b7ea.', STARTKEY => '', ENDKEY => ''} 2024-11-23T21:54:44,505 DEBUG [RS_OPEN_REGION-regionserver/adda16083778:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testBlockEvictionOnGracefulStop1732398884153 d1d01dcb0a94d6c09a6bd0b1d689b7ea 2024-11-23T21:54:44,506 DEBUG [RS_OPEN_REGION-regionserver/adda16083778:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(898): Instantiated testBlockEvictionOnGracefulStop1732398884153,,1732398884158.d1d01dcb0a94d6c09a6bd0b1d689b7ea.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-23T21:54:44,506 DEBUG [RS_OPEN_REGION-regionserver/adda16083778:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7794): checking encryption for d1d01dcb0a94d6c09a6bd0b1d689b7ea 2024-11-23T21:54:44,506 DEBUG [RS_OPEN_REGION-regionserver/adda16083778:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7797): checking classloading for d1d01dcb0a94d6c09a6bd0b1d689b7ea 2024-11-23T21:54:44,507 INFO [StoreOpener-d1d01dcb0a94d6c09a6bd0b1d689b7ea-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family family of region d1d01dcb0a94d6c09a6bd0b1d689b7ea 2024-11-23T21:54:44,510 INFO [StoreOpener-d1d01dcb0a94d6c09a6bd0b1d689b7ea-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region d1d01dcb0a94d6c09a6bd0b1d689b7ea columnFamilyName family 2024-11-23T21:54:44,510 DEBUG [StoreOpener-d1d01dcb0a94d6c09a6bd0b1d689b7ea-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T21:54:44,510 INFO [StoreOpener-d1d01dcb0a94d6c09a6bd0b1d689b7ea-1 {}] regionserver.HStore(327): Store=d1d01dcb0a94d6c09a6bd0b1d689b7ea/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-23T21:54:44,511 DEBUG [RS_OPEN_REGION-regionserver/adda16083778:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1038): replaying wal for d1d01dcb0a94d6c09a6bd0b1d689b7ea 2024-11-23T21:54:44,511 DEBUG [RS_OPEN_REGION-regionserver/adda16083778:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/93be71cd-4dcf-2478-daf8-a2306a879141/e0862929-e242-795d-1e0b-fe7766981a5b/data/default/testBlockEvictionOnGracefulStop1732398884153/d1d01dcb0a94d6c09a6bd0b1d689b7ea 2024-11-23T21:54:44,511 DEBUG [RS_OPEN_REGION-regionserver/adda16083778:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/93be71cd-4dcf-2478-daf8-a2306a879141/e0862929-e242-795d-1e0b-fe7766981a5b/data/default/testBlockEvictionOnGracefulStop1732398884153/d1d01dcb0a94d6c09a6bd0b1d689b7ea 2024-11-23T21:54:44,512 DEBUG [RS_OPEN_REGION-regionserver/adda16083778:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1048): stopping wal replay for d1d01dcb0a94d6c09a6bd0b1d689b7ea 2024-11-23T21:54:44,512 DEBUG [RS_OPEN_REGION-regionserver/adda16083778:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1060): Cleaning up temporary data for d1d01dcb0a94d6c09a6bd0b1d689b7ea 2024-11-23T21:54:44,512 DEBUG [RS_OPEN_REGION-regionserver/adda16083778:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1093): writing seq id for d1d01dcb0a94d6c09a6bd0b1d689b7ea 2024-11-23T21:54:44,517 DEBUG [RS_OPEN_REGION-regionserver/adda16083778:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/93be71cd-4dcf-2478-daf8-a2306a879141/e0862929-e242-795d-1e0b-fe7766981a5b/data/default/testBlockEvictionOnGracefulStop1732398884153/d1d01dcb0a94d6c09a6bd0b1d689b7ea/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-23T21:54:44,518 INFO [RS_OPEN_REGION-regionserver/adda16083778:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1114): Opened d1d01dcb0a94d6c09a6bd0b1d689b7ea; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=59594344, jitterRate=-0.11197507381439209}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-23T21:54:44,518 DEBUG [RS_OPEN_REGION-regionserver/adda16083778:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1122): Running coprocessor post-open hooks for d1d01dcb0a94d6c09a6bd0b1d689b7ea 2024-11-23T21:54:44,519 DEBUG [RS_OPEN_REGION-regionserver/adda16083778:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1006): Region open journal for d1d01dcb0a94d6c09a6bd0b1d689b7ea: Running coprocessor pre-open hook at 1732398884506Writing region info on filesystem at 1732398884506Initializing all the Stores at 1732398884506Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732398884506Cleaning up temporary data from old regions at 1732398884512 (+6 ms)Running coprocessor post-open hooks at 1732398884518 (+6 ms)Region opened successfully at 1732398884519 (+1 ms) 2024-11-23T21:54:44,520 INFO [RS_OPEN_REGION-regionserver/adda16083778:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2236): Post open deploy tasks for testBlockEvictionOnGracefulStop1732398884153,,1732398884158.d1d01dcb0a94d6c09a6bd0b1d689b7ea., pid=6, masterSystemTime=1732398884503 2024-11-23T21:54:44,522 DEBUG [RS_OPEN_REGION-regionserver/adda16083778:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2266): Finished post open deploy task for testBlockEvictionOnGracefulStop1732398884153,,1732398884158.d1d01dcb0a94d6c09a6bd0b1d689b7ea. 2024-11-23T21:54:44,523 INFO [RS_OPEN_REGION-regionserver/adda16083778:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(153): Opened testBlockEvictionOnGracefulStop1732398884153,,1732398884158.d1d01dcb0a94d6c09a6bd0b1d689b7ea. 2024-11-23T21:54:44,523 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=d1d01dcb0a94d6c09a6bd0b1d689b7ea, regionState=OPEN, openSeqNum=2, regionLocation=adda16083778,37831,1732398882981 2024-11-23T21:54:44,526 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-8-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure d1d01dcb0a94d6c09a6bd0b1d689b7ea, server=adda16083778,37831,1732398882981 because future has completed 2024-11-23T21:54:44,529 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=6, resume processing ppid=5 2024-11-23T21:54:44,529 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure d1d01dcb0a94d6c09a6bd0b1d689b7ea, server=adda16083778,37831,1732398882981 in 178 msec 2024-11-23T21:54:44,531 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=5, resume processing ppid=4 2024-11-23T21:54:44,531 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1732398884153, region=d1d01dcb0a94d6c09a6bd0b1d689b7ea, ASSIGN in 341 msec 2024-11-23T21:54:44,531 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=testBlockEvictionOnGracefulStop1732398884153 execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-23T21:54:44,532 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testBlockEvictionOnGracefulStop1732398884153","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732398884532"}]},"ts":"1732398884532"} 2024-11-23T21:54:44,534 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=testBlockEvictionOnGracefulStop1732398884153, state=ENABLED in hbase:meta 2024-11-23T21:54:44,535 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=testBlockEvictionOnGracefulStop1732398884153 execute state=CREATE_TABLE_POST_OPERATION 2024-11-23T21:54:44,537 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=testBlockEvictionOnGracefulStop1732398884153 in 377 msec 2024-11-23T21:54:44,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33953 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-23T21:54:44,788 INFO [RPCClient-NioEventLoopGroup-5-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:testBlockEvictionOnGracefulStop1732398884153 completed 2024-11-23T21:54:44,788 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table testBlockEvictionOnGracefulStop1732398884153 get assigned. Timeout = 60000ms 2024-11-23T21:54:44,789 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-23T21:54:44,792 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table testBlockEvictionOnGracefulStop1732398884153 assigned to meta. Checking AM states. 2024-11-23T21:54:44,792 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-23T21:54:44,792 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table testBlockEvictionOnGracefulStop1732398884153 assigned. 2024-11-23T21:54:44,796 DEBUG [RPCClient-NioEventLoopGroup-5-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testBlockEvictionOnGracefulStop1732398884153', row='row1', locateType=CURRENT is [region=testBlockEvictionOnGracefulStop1732398884153,,1732398884158.d1d01dcb0a94d6c09a6bd0b1d689b7ea., hostname=adda16083778,37831,1732398882981, seqNum=2] 2024-11-23T21:54:44,800 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing d1d01dcb0a94d6c09a6bd0b1d689b7ea 1/1 column families, dataSize=78 B heapSize=480 B 2024-11-23T21:54:44,809 DEBUG [Time-limited test {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on a9be9bf9abe44a04b4ff30a70e5e96f6 2024-11-23T21:54:44,820 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/93be71cd-4dcf-2478-daf8-a2306a879141/e0862929-e242-795d-1e0b-fe7766981a5b/data/default/testBlockEvictionOnGracefulStop1732398884153/d1d01dcb0a94d6c09a6bd0b1d689b7ea/.tmp/family/a9be9bf9abe44a04b4ff30a70e5e96f6 is 43, key is row1/family:qf1/1/Put/seqid=0 2024-11-23T21:54:44,820 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=78 B at sequenceid=6 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/93be71cd-4dcf-2478-daf8-a2306a879141/e0862929-e242-795d-1e0b-fe7766981a5b/data/default/testBlockEvictionOnGracefulStop1732398884153/d1d01dcb0a94d6c09a6bd0b1d689b7ea/.tmp/family/a9be9bf9abe44a04b4ff30a70e5e96f6 2024-11-23T21:54:44,823 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/93be71cd-4dcf-2478-daf8-a2306a879141/e0862929-e242-795d-1e0b-fe7766981a5b/data/default/testBlockEvictionOnGracefulStop1732398884153/d1d01dcb0a94d6c09a6bd0b1d689b7ea/.tmp/family/a9be9bf9abe44a04b4ff30a70e5e96f6 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/93be71cd-4dcf-2478-daf8-a2306a879141/e0862929-e242-795d-1e0b-fe7766981a5b/data/default/testBlockEvictionOnGracefulStop1732398884153/d1d01dcb0a94d6c09a6bd0b1d689b7ea/family/a9be9bf9abe44a04b4ff30a70e5e96f6 2024-11-23T21:54:44,823 DEBUG [Time-limited test {}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/93be71cd-4dcf-2478-daf8-a2306a879141/e0862929-e242-795d-1e0b-fe7766981a5b/data/default/testBlockEvictionOnGracefulStop1732398884153/d1d01dcb0a94d6c09a6bd0b1d689b7ea/family/a9be9bf9abe44a04b4ff30a70e5e96f6, delay=947 ms 2024-11-23T21:54:44,824 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/93be71cd-4dcf-2478-daf8-a2306a879141/e0862929-e242-795d-1e0b-fe7766981a5b/data/default/testBlockEvictionOnGracefulStop1732398884153/d1d01dcb0a94d6c09a6bd0b1d689b7ea/family/a9be9bf9abe44a04b4ff30a70e5e96f6, entries=2, sequenceid=6, filesize=4.9 K 2024-11-23T21:54:44,825 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~78 B/78, heapSize ~464 B/464, currentSize=0 B/0 for d1d01dcb0a94d6c09a6bd0b1d689b7ea in 25ms, sequenceid=6, compaction requested=false 2024-11-23T21:54:44,825 DEBUG [Time-limited test {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'testBlockEvictionOnGracefulStop1732398884153' 2024-11-23T21:54:44,826 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for d1d01dcb0a94d6c09a6bd0b1d689b7ea: 2024-11-23T21:54:44,903 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-23T21:54:45,772 DEBUG [hfile-prefetch-1732398880564 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key a9be9bf9abe44a04b4ff30a70e5e96f6_0. 2024-11-23T21:54:45,773 DEBUG [hfile-prefetch-1732398880564 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key a9be9bf9abe44a04b4ff30a70e5e96f6_117. 2024-11-23T21:54:45,773 DEBUG [hfile-prefetch-1732398880564 {}] bucket.BucketCache(2281): Notifying caching completed for file file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/93be71cd-4dcf-2478-daf8-a2306a879141/e0862929-e242-795d-1e0b-fe7766981a5b/data/default/testBlockEvictionOnGracefulStop1732398884153/d1d01dcb0a94d6c09a6bd0b1d689b7ea/family/a9be9bf9abe44a04b4ff30a70e5e96f6, with total blocks 2, and data blocks 1 2024-11-23T21:54:45,773 DEBUG [hfile-prefetch-1732398880564 {}] bucket.BucketCache(2285): iterating over 0 entries in the backing map 2024-11-23T21:54:45,773 DEBUG [hfile-prefetch-1732398880564 {}] bucket.BucketCache(2308): Prefetch executor completed for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/93be71cd-4dcf-2478-daf8-a2306a879141/e0862929-e242-795d-1e0b-fe7766981a5b/data/default/testBlockEvictionOnGracefulStop1732398884153/d1d01dcb0a94d6c09a6bd0b1d689b7ea/family/a9be9bf9abe44a04b4ff30a70e5e96f6, but only 0 data blocks were cached. Total data blocks for file: 1. Checking for blocks pending cache in cache writer queue. 2024-11-23T21:54:45,774 DEBUG [hfile-prefetch-1732398880564 {}] bucket.BucketCache(2318): There are still blocks pending caching for file file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/93be71cd-4dcf-2478-daf8-a2306a879141/e0862929-e242-795d-1e0b-fe7766981a5b/data/default/testBlockEvictionOnGracefulStop1732398884153/d1d01dcb0a94d6c09a6bd0b1d689b7ea/family/a9be9bf9abe44a04b4ff30a70e5e96f6. Will sleep 100ms and try the verification again. 2024-11-23T21:54:45,826 INFO [Time-limited test {}] hbase.SingleProcessHBaseCluster(504): Stopping Thread[RS:0;adda16083778:37831,5,FailOnTimeoutGroup] 2024-11-23T21:54:45,827 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server 'adda16083778,37831,1732398882981' ***** 2024-11-23T21:54:45,827 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Stopping rs 0 2024-11-23T21:54:45,827 INFO [RS:0;adda16083778:37831 {}] bucket.BucketCache(1738): Disabling cache 2024-11-23T21:54:45,864 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-23T21:54:45,864 INFO [regionserver/adda16083778:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-23T21:54:45,874 DEBUG [hfile-prefetch-1732398880564 {}] bucket.BucketCache(2281): Notifying caching completed for file file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/93be71cd-4dcf-2478-daf8-a2306a879141/e0862929-e242-795d-1e0b-fe7766981a5b/data/default/testBlockEvictionOnGracefulStop1732398884153/d1d01dcb0a94d6c09a6bd0b1d689b7ea/family/a9be9bf9abe44a04b4ff30a70e5e96f6, with total blocks 2, and data blocks 1 2024-11-23T21:54:45,874 DEBUG [hfile-prefetch-1732398880564 {}] bucket.BucketCache(2285): iterating over 0 entries in the backing map 2024-11-23T21:54:45,874 DEBUG [hfile-prefetch-1732398880564 {}] bucket.BucketCache(2308): Prefetch executor completed for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/93be71cd-4dcf-2478-daf8-a2306a879141/e0862929-e242-795d-1e0b-fe7766981a5b/data/default/testBlockEvictionOnGracefulStop1732398884153/d1d01dcb0a94d6c09a6bd0b1d689b7ea/family/a9be9bf9abe44a04b4ff30a70e5e96f6, but only 0 data blocks were cached. Total data blocks for file: 1. Checking for blocks pending cache in cache writer queue. 2024-11-23T21:54:45,875 DEBUG [hfile-prefetch-1732398880564 {}] bucket.BucketCache(2318): There are still blocks pending caching for file file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/93be71cd-4dcf-2478-daf8-a2306a879141/e0862929-e242-795d-1e0b-fe7766981a5b/data/default/testBlockEvictionOnGracefulStop1732398884153/d1d01dcb0a94d6c09a6bd0b1d689b7ea/family/a9be9bf9abe44a04b4ff30a70e5e96f6. Will sleep 100ms and try the verification again. 2024-11-23T21:54:45,975 DEBUG [hfile-prefetch-1732398880564 {}] bucket.BucketCache(2281): Notifying caching completed for file file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/93be71cd-4dcf-2478-daf8-a2306a879141/e0862929-e242-795d-1e0b-fe7766981a5b/data/default/testBlockEvictionOnGracefulStop1732398884153/d1d01dcb0a94d6c09a6bd0b1d689b7ea/family/a9be9bf9abe44a04b4ff30a70e5e96f6, with total blocks 2, and data blocks 1 2024-11-23T21:54:45,975 DEBUG [hfile-prefetch-1732398880564 {}] bucket.BucketCache(2285): iterating over 0 entries in the backing map 2024-11-23T21:54:45,975 DEBUG [hfile-prefetch-1732398880564 {}] bucket.BucketCache(2308): Prefetch executor completed for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/93be71cd-4dcf-2478-daf8-a2306a879141/e0862929-e242-795d-1e0b-fe7766981a5b/data/default/testBlockEvictionOnGracefulStop1732398884153/d1d01dcb0a94d6c09a6bd0b1d689b7ea/family/a9be9bf9abe44a04b4ff30a70e5e96f6, but only 0 data blocks were cached. Total data blocks for file: 1. Checking for blocks pending cache in cache writer queue. 2024-11-23T21:54:45,975 DEBUG [hfile-prefetch-1732398880564 {}] bucket.BucketCache(2318): There are still blocks pending caching for file file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/93be71cd-4dcf-2478-daf8-a2306a879141/e0862929-e242-795d-1e0b-fe7766981a5b/data/default/testBlockEvictionOnGracefulStop1732398884153/d1d01dcb0a94d6c09a6bd0b1d689b7ea/family/a9be9bf9abe44a04b4ff30a70e5e96f6. Will sleep 100ms and try the verification again. 2024-11-23T21:54:45,976 INFO [Time-limited test-BucketCacheWriter-0 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-0 exiting, cacheEnabled=false 2024-11-23T21:54:45,976 INFO [Time-limited test-BucketCacheWriter-1 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-1 exiting, cacheEnabled=false 2024-11-23T21:54:45,976 INFO [RS:0;adda16083778:37831 {}] bucket.BucketCache(1762): Shutdown bucket cache: IO persistent=true; path to write=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d246ebad-447b-0ccb-984f-a43056e9385e/bucket.persistence 2024-11-23T21:54:45,976 INFO [Time-limited test-BucketCacheWriter-2 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-2 exiting, cacheEnabled=false 2024-11-23T21:54:45,976 INFO [RS:0;adda16083778:37831 {}] bucket.BucketCache(1768): Shutting down cache persister thread. 2024-11-23T21:54:45,977 WARN [bucket-cache-persister {}] bucket.BucketCachePersister(64): Interrupting BucketCachePersister thread. java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCachePersister.run(BucketCachePersister.java:45) ~[classes/:?] 2024-11-23T21:54:45,987 DEBUG [RS:0;adda16083778:37831 {}] bucket.BucketCache(1487): Thread RS:0;adda16083778:37831 started persisting bucket cache to file 2024-11-23T21:54:45,987 DEBUG [RS:0;adda16083778:37831 {}] bucket.BucketCache(1494): Persist in new chunked persistence format. 2024-11-23T21:54:45,987 DEBUG [RS:0;adda16083778:37831 {}] bucket.BucketCache(1673): persistToFile: before persisting backing map size: 1, fullycachedFiles size: 0, chunkSize: 10000 2024-11-23T21:54:45,992 DEBUG [RS:0;adda16083778:37831 {}] bucket.BucketCache(1680): persistToFile: after persisting backing map size: 1, fullycachedFiles size: 0 2024-11-23T21:54:45,993 DEBUG [RS:0;adda16083778:37831 {}] bucket.BucketCache(1498): PersistToFile: after persisting backing map size: 1, fullycachedFiles size: 0, file name: bucket.persistence1732398885987 2024-11-23T21:54:45,993 DEBUG [RS:0;adda16083778:37831 {}] bucket.BucketCache(1509): Thread RS:0;adda16083778:37831 finished persisting bucket cache to file, renaming 2024-11-23T21:54:45,993 INFO [RS:0;adda16083778:37831 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-23T21:54:45,993 INFO [RS:0;adda16083778:37831 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-23T21:54:45,993 INFO [RS:0;adda16083778:37831 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-23T21:54:45,994 INFO [RS:0;adda16083778:37831 {}] regionserver.HRegionServer(3091): Received CLOSE for d1d01dcb0a94d6c09a6bd0b1d689b7ea 2024-11-23T21:54:45,994 INFO [RS:0;adda16083778:37831 {}] regionserver.HRegionServer(959): stopping server adda16083778,37831,1732398882981 2024-11-23T21:54:45,994 INFO [RS:0;adda16083778:37831 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-23T21:54:45,994 INFO [RS:0;adda16083778:37831 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;adda16083778:37831. 2024-11-23T21:54:45,994 INFO [RS_CLOSE_REGION-regionserver/adda16083778:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1662): Closing region testBlockEvictionOnGracefulStop1732398884153,,1732398884158.d1d01dcb0a94d6c09a6bd0b1d689b7ea. during a graceful stop, and cache persistence is on, so setting evict on close to false. 2024-11-23T21:54:45,995 DEBUG [RS_CLOSE_REGION-regionserver/adda16083778:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing d1d01dcb0a94d6c09a6bd0b1d689b7ea, disabling compactions & flushes 2024-11-23T21:54:45,995 DEBUG [RS:0;adda16083778:37831 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-23T21:54:45,995 INFO [RS_CLOSE_REGION-regionserver/adda16083778:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testBlockEvictionOnGracefulStop1732398884153,,1732398884158.d1d01dcb0a94d6c09a6bd0b1d689b7ea. 2024-11-23T21:54:45,995 DEBUG [RS:0;adda16083778:37831 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T21:54:45,995 DEBUG [RS_CLOSE_REGION-regionserver/adda16083778:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testBlockEvictionOnGracefulStop1732398884153,,1732398884158.d1d01dcb0a94d6c09a6bd0b1d689b7ea. 2024-11-23T21:54:45,995 DEBUG [RS_CLOSE_REGION-regionserver/adda16083778:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testBlockEvictionOnGracefulStop1732398884153,,1732398884158.d1d01dcb0a94d6c09a6bd0b1d689b7ea. after waiting 0 ms 2024-11-23T21:54:45,995 INFO [RS:0;adda16083778:37831 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-23T21:54:45,995 DEBUG [RS_CLOSE_REGION-regionserver/adda16083778:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testBlockEvictionOnGracefulStop1732398884153,,1732398884158.d1d01dcb0a94d6c09a6bd0b1d689b7ea. 2024-11-23T21:54:45,995 INFO [RS:0;adda16083778:37831 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-23T21:54:45,995 INFO [RS:0;adda16083778:37831 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-23T21:54:45,996 INFO [RS:0;adda16083778:37831 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-11-23T21:54:45,996 INFO [RS:0;adda16083778:37831 {}] regionserver.HRegionServer(1321): Waiting on 2 regions to close 2024-11-23T21:54:45,996 DEBUG [RS:0;adda16083778:37831 {}] regionserver.HRegionServer(1325): Online Regions={1588230740=hbase:meta,,1.1588230740, d1d01dcb0a94d6c09a6bd0b1d689b7ea=testBlockEvictionOnGracefulStop1732398884153,,1732398884158.d1d01dcb0a94d6c09a6bd0b1d689b7ea.} 2024-11-23T21:54:45,996 DEBUG [RS:0;adda16083778:37831 {}] regionserver.HRegionServer(1351): Waiting on 1588230740, d1d01dcb0a94d6c09a6bd0b1d689b7ea 2024-11-23T21:54:45,996 INFO [RS_CLOSE_META-regionserver/adda16083778:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1662): Closing region hbase:meta,,1.1588230740 during a graceful stop, and cache persistence is on, so setting evict on close to false. 2024-11-23T21:54:45,996 DEBUG [StoreFileCloser-d1d01dcb0a94d6c09a6bd0b1d689b7ea-family-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/93be71cd-4dcf-2478-daf8-a2306a879141/e0862929-e242-795d-1e0b-fe7766981a5b/data/default/testBlockEvictionOnGracefulStop1732398884153/d1d01dcb0a94d6c09a6bd0b1d689b7ea/family/a9be9bf9abe44a04b4ff30a70e5e96f6 2024-11-23T21:54:45,997 DEBUG [RS_CLOSE_META-regionserver/adda16083778:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-23T21:54:45,997 INFO [RS_CLOSE_META-regionserver/adda16083778:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-23T21:54:45,997 DEBUG [RS_CLOSE_META-regionserver/adda16083778:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-23T21:54:45,997 DEBUG [RS_CLOSE_META-regionserver/adda16083778:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-23T21:54:45,997 DEBUG [RS_CLOSE_META-regionserver/adda16083778:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-23T21:54:45,998 INFO [RS_CLOSE_META-regionserver/adda16083778:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=1.75 KB heapSize=3.78 KB 2024-11-23T21:54:46,000 WARN [hfile-prefetch-1732398880564 {}] hfile.HFilePreadReader$1(158): Prefetch path=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/93be71cd-4dcf-2478-daf8-a2306a879141/e0862929-e242-795d-1e0b-fe7766981a5b/data/default/testBlockEvictionOnGracefulStop1732398884153/d1d01dcb0a94d6c09a6bd0b1d689b7ea/family/a9be9bf9abe44a04b4ff30a70e5e96f6, offset=158, end=158 java.lang.RuntimeException: java.lang.InterruptedException: sleep interrupted at org.apache.hadoop.hbase.io.hfile.bucket.BucketCache.notifyFileCachingCompleted(BucketCache.java:2337) ~[classes/:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCache.notifyFileCachingCompleted(BucketCache.java:2321) ~[classes/:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCache.notifyFileCachingCompleted(BucketCache.java:2321) ~[classes/:?] at org.apache.hadoop.hbase.io.hfile.CombinedBlockCache.notifyFileCachingCompleted(CombinedBlockCache.java:469) ~[classes/:?] at org.apache.hadoop.hbase.io.hfile.HFilePreadReader$1.run(HFilePreadReader.java:148) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.lambda$tracedRunnable$2(TraceUtil.java:155) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:539) ~[?:?] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:304) ~[?:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCache.notifyFileCachingCompleted(BucketCache.java:2320) ~[classes/:?] ... 11 more 2024-11-23T21:54:46,001 DEBUG [hfile-prefetch-1732398880564 {}] hfile.PrefetchExecutor(121): Prefetch completed for a9be9bf9abe44a04b4ff30a70e5e96f6 2024-11-23T21:54:46,003 DEBUG [RS_CLOSE_REGION-regionserver/adda16083778:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/93be71cd-4dcf-2478-daf8-a2306a879141/e0862929-e242-795d-1e0b-fe7766981a5b/data/default/testBlockEvictionOnGracefulStop1732398884153/d1d01dcb0a94d6c09a6bd0b1d689b7ea/recovered.edits/9.seqid, newMaxSeqId=9, maxSeqId=1 2024-11-23T21:54:46,003 INFO [RS_CLOSE_REGION-regionserver/adda16083778:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testBlockEvictionOnGracefulStop1732398884153,,1732398884158.d1d01dcb0a94d6c09a6bd0b1d689b7ea. 2024-11-23T21:54:46,003 DEBUG [RS_CLOSE_REGION-regionserver/adda16083778:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for d1d01dcb0a94d6c09a6bd0b1d689b7ea: Waiting for close lock at 1732398885994Running coprocessor pre-close hooks at 1732398885995 (+1 ms)Disabling compacts and flushes for region at 1732398885995Disabling writes for close at 1732398885995Writing region close event to WAL at 1732398885997 (+2 ms)Running coprocessor post-close hooks at 1732398886003 (+6 ms)Closed at 1732398886003 2024-11-23T21:54:46,003 DEBUG [RS_CLOSE_REGION-regionserver/adda16083778:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testBlockEvictionOnGracefulStop1732398884153,,1732398884158.d1d01dcb0a94d6c09a6bd0b1d689b7ea. 2024-11-23T21:54:46,008 DEBUG [RS_CLOSE_META-regionserver/adda16083778:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 1b70fd2dcd8f449b823e9ebf60be2907 2024-11-23T21:54:46,020 DEBUG [RS_CLOSE_META-regionserver/adda16083778:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/93be71cd-4dcf-2478-daf8-a2306a879141/e0862929-e242-795d-1e0b-fe7766981a5b/data/hbase/meta/1588230740/.tmp/info/1b70fd2dcd8f449b823e9ebf60be2907 is 209, key is testBlockEvictionOnGracefulStop1732398884153,,1732398884158.d1d01dcb0a94d6c09a6bd0b1d689b7ea./info:regioninfo/1732398884523/Put/seqid=0 2024-11-23T21:54:46,021 INFO [RS_CLOSE_META-regionserver/adda16083778:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.53 KB at sequenceid=11 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/93be71cd-4dcf-2478-daf8-a2306a879141/e0862929-e242-795d-1e0b-fe7766981a5b/data/hbase/meta/1588230740/.tmp/info/1b70fd2dcd8f449b823e9ebf60be2907 2024-11-23T21:54:46,028 DEBUG [RS_CLOSE_META-regionserver/adda16083778:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFile$WriterFactory(308): Unable to set drop behind on cfd1bd3e99364567803dec829e99ee1a 2024-11-23T21:54:46,031 DEBUG [bucket-cache-persister {}] bucket.BucketCachePersister(47): Cache is inconsistent, persisting to disk 2024-11-23T21:54:46,031 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1487): Thread bucket-cache-persister started persisting bucket cache to file 2024-11-23T21:54:46,031 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1494): Persist in new chunked persistence format. 2024-11-23T21:54:46,031 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1673): persistToFile: before persisting backing map size: 0, fullycachedFiles size: 0, chunkSize: 10000 2024-11-23T21:54:46,034 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1680): persistToFile: after persisting backing map size: 0, fullycachedFiles size: 0 2024-11-23T21:54:46,034 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1498): PersistToFile: after persisting backing map size: 0, fullycachedFiles size: 0, file name: bucket.persistence1732398886031 2024-11-23T21:54:46,034 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1509): Thread bucket-cache-persister finished persisting bucket cache to file, renaming 2024-11-23T21:54:46,041 DEBUG [RS_CLOSE_META-regionserver/adda16083778:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/93be71cd-4dcf-2478-daf8-a2306a879141/e0862929-e242-795d-1e0b-fe7766981a5b/data/hbase/meta/1588230740/.tmp/ns/cfd1bd3e99364567803dec829e99ee1a is 43, key is default/ns:d/1732398884115/Put/seqid=0 2024-11-23T21:54:46,042 INFO [RS_CLOSE_META-regionserver/adda16083778:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=74 B at sequenceid=11 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/93be71cd-4dcf-2478-daf8-a2306a879141/e0862929-e242-795d-1e0b-fe7766981a5b/data/hbase/meta/1588230740/.tmp/ns/cfd1bd3e99364567803dec829e99ee1a 2024-11-23T21:54:46,049 DEBUG [RS_CLOSE_META-regionserver/adda16083778:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 74a27669f3a94e7cbe5394a8064aaee3 2024-11-23T21:54:46,060 DEBUG [RS_CLOSE_META-regionserver/adda16083778:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/93be71cd-4dcf-2478-daf8-a2306a879141/e0862929-e242-795d-1e0b-fe7766981a5b/data/hbase/meta/1588230740/.tmp/table/74a27669f3a94e7cbe5394a8064aaee3 is 80, key is testBlockEvictionOnGracefulStop1732398884153/table:state/1732398884532/Put/seqid=0 2024-11-23T21:54:46,060 INFO [RS_CLOSE_META-regionserver/adda16083778:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=152 B at sequenceid=11 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/93be71cd-4dcf-2478-daf8-a2306a879141/e0862929-e242-795d-1e0b-fe7766981a5b/data/hbase/meta/1588230740/.tmp/table/74a27669f3a94e7cbe5394a8064aaee3 2024-11-23T21:54:46,063 DEBUG [RS_CLOSE_META-regionserver/adda16083778:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/93be71cd-4dcf-2478-daf8-a2306a879141/e0862929-e242-795d-1e0b-fe7766981a5b/data/hbase/meta/1588230740/.tmp/info/1b70fd2dcd8f449b823e9ebf60be2907 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/93be71cd-4dcf-2478-daf8-a2306a879141/e0862929-e242-795d-1e0b-fe7766981a5b/data/hbase/meta/1588230740/info/1b70fd2dcd8f449b823e9ebf60be2907 2024-11-23T21:54:46,064 DEBUG [RS_CLOSE_META-regionserver/adda16083778:0-0 {event_type=M_RS_CLOSE_META}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/93be71cd-4dcf-2478-daf8-a2306a879141/e0862929-e242-795d-1e0b-fe7766981a5b/data/hbase/meta/1588230740/info/1b70fd2dcd8f449b823e9ebf60be2907, delay=980 ms 2024-11-23T21:54:46,065 INFO [RS_CLOSE_META-regionserver/adda16083778:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/93be71cd-4dcf-2478-daf8-a2306a879141/e0862929-e242-795d-1e0b-fe7766981a5b/data/hbase/meta/1588230740/info/1b70fd2dcd8f449b823e9ebf60be2907, entries=10, sequenceid=11, filesize=7.0 K 2024-11-23T21:54:46,065 DEBUG [RS_CLOSE_META-regionserver/adda16083778:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/93be71cd-4dcf-2478-daf8-a2306a879141/e0862929-e242-795d-1e0b-fe7766981a5b/data/hbase/meta/1588230740/.tmp/ns/cfd1bd3e99364567803dec829e99ee1a as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/93be71cd-4dcf-2478-daf8-a2306a879141/e0862929-e242-795d-1e0b-fe7766981a5b/data/hbase/meta/1588230740/ns/cfd1bd3e99364567803dec829e99ee1a 2024-11-23T21:54:46,066 DEBUG [RS_CLOSE_META-regionserver/adda16083778:0-0 {event_type=M_RS_CLOSE_META}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/93be71cd-4dcf-2478-daf8-a2306a879141/e0862929-e242-795d-1e0b-fe7766981a5b/data/hbase/meta/1588230740/ns/cfd1bd3e99364567803dec829e99ee1a, delay=921 ms 2024-11-23T21:54:46,067 INFO [RS_CLOSE_META-regionserver/adda16083778:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/93be71cd-4dcf-2478-daf8-a2306a879141/e0862929-e242-795d-1e0b-fe7766981a5b/data/hbase/meta/1588230740/ns/cfd1bd3e99364567803dec829e99ee1a, entries=2, sequenceid=11, filesize=5.0 K 2024-11-23T21:54:46,068 DEBUG [RS_CLOSE_META-regionserver/adda16083778:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/93be71cd-4dcf-2478-daf8-a2306a879141/e0862929-e242-795d-1e0b-fe7766981a5b/data/hbase/meta/1588230740/.tmp/table/74a27669f3a94e7cbe5394a8064aaee3 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/93be71cd-4dcf-2478-daf8-a2306a879141/e0862929-e242-795d-1e0b-fe7766981a5b/data/hbase/meta/1588230740/table/74a27669f3a94e7cbe5394a8064aaee3 2024-11-23T21:54:46,069 DEBUG [RS_CLOSE_META-regionserver/adda16083778:0-0 {event_type=M_RS_CLOSE_META}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/93be71cd-4dcf-2478-daf8-a2306a879141/e0862929-e242-795d-1e0b-fe7766981a5b/data/hbase/meta/1588230740/table/74a27669f3a94e7cbe5394a8064aaee3, delay=969 ms 2024-11-23T21:54:46,069 INFO [RS_CLOSE_META-regionserver/adda16083778:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/93be71cd-4dcf-2478-daf8-a2306a879141/e0862929-e242-795d-1e0b-fe7766981a5b/data/hbase/meta/1588230740/table/74a27669f3a94e7cbe5394a8064aaee3, entries=2, sequenceid=11, filesize=5.3 K 2024-11-23T21:54:46,070 INFO [RS_CLOSE_META-regionserver/adda16083778:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~1.75 KB/1797, heapSize ~3.48 KB/3568, currentSize=0 B/0 for 1588230740 in 73ms, sequenceid=11, compaction requested=false 2024-11-23T21:54:46,071 DEBUG [StoreFileCloser-1588230740-info-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/93be71cd-4dcf-2478-daf8-a2306a879141/e0862929-e242-795d-1e0b-fe7766981a5b/data/hbase/meta/1588230740/info/1b70fd2dcd8f449b823e9ebf60be2907 2024-11-23T21:54:46,071 DEBUG [StoreFileCloser-1588230740-ns-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/93be71cd-4dcf-2478-daf8-a2306a879141/e0862929-e242-795d-1e0b-fe7766981a5b/data/hbase/meta/1588230740/ns/cfd1bd3e99364567803dec829e99ee1a 2024-11-23T21:54:46,072 DEBUG [StoreFileCloser-1588230740-table-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/93be71cd-4dcf-2478-daf8-a2306a879141/e0862929-e242-795d-1e0b-fe7766981a5b/data/hbase/meta/1588230740/table/74a27669f3a94e7cbe5394a8064aaee3 2024-11-23T21:54:46,076 DEBUG [RS_CLOSE_META-regionserver/adda16083778:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/93be71cd-4dcf-2478-daf8-a2306a879141/e0862929-e242-795d-1e0b-fe7766981a5b/data/hbase/meta/1588230740/recovered.edits/14.seqid, newMaxSeqId=14, maxSeqId=1 2024-11-23T21:54:46,076 DEBUG [RS_CLOSE_META-regionserver/adda16083778:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-23T21:54:46,076 INFO [RS_CLOSE_META-regionserver/adda16083778:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-23T21:54:46,076 DEBUG [RS_CLOSE_META-regionserver/adda16083778:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1732398885996Running coprocessor pre-close hooks at 1732398885996Disabling compacts and flushes for region at 1732398885997 (+1 ms)Disabling writes for close at 1732398885997Obtaining lock to block concurrent updates at 1732398885998 (+1 ms)Preparing flush snapshotting stores in 1588230740 at 1732398885998Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=1797, getHeapSize=3808, getOffHeapSize=0, getCellsCount=14 at 1732398885998Flushing stores of hbase:meta,,1.1588230740 at 1732398885999 (+1 ms)Flushing 1588230740/info: creating writer at 1732398885999Flushing 1588230740/info: appending metadata at 1732398886020 (+21 ms)Flushing 1588230740/info: closing flushed file at 1732398886020Flushing 1588230740/ns: creating writer at 1732398886023 (+3 ms)Flushing 1588230740/ns: appending metadata at 1732398886041 (+18 ms)Flushing 1588230740/ns: closing flushed file at 1732398886041Flushing 1588230740/table: creating writer at 1732398886044 (+3 ms)Flushing 1588230740/table: appending metadata at 1732398886059 (+15 ms)Flushing 1588230740/table: closing flushed file at 1732398886059Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@45954005: reopening flushed file at 1732398886062 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@718b9c29: reopening flushed file at 1732398886065 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@dcf60be: reopening flushed file at 1732398886067 (+2 ms)Finished flush of dataSize ~1.75 KB/1797, heapSize ~3.48 KB/3568, currentSize=0 B/0 for 1588230740 in 73ms, sequenceid=11, compaction requested=false at 1732398886070 (+3 ms)Writing region close event to WAL at 1732398886072 (+2 ms)Running coprocessor post-close hooks at 1732398886076 (+4 ms)Closed at 1732398886076 2024-11-23T21:54:46,077 DEBUG [RS_CLOSE_META-regionserver/adda16083778:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-11-23T21:54:46,197 INFO [RS:0;adda16083778:37831 {}] regionserver.HRegionServer(976): stopping server adda16083778,37831,1732398882981; all regions closed. 2024-11-23T21:54:46,202 DEBUG [RS:0;adda16083778:37831 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/93be71cd-4dcf-2478-daf8-a2306a879141/e0862929-e242-795d-1e0b-fe7766981a5b/oldWALs 2024-11-23T21:54:46,203 INFO [RS:0;adda16083778:37831 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL adda16083778%2C37831%2C1732398882981.meta:.meta(num 1732398884042) 2024-11-23T21:54:46,207 DEBUG [RS:0;adda16083778:37831 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/93be71cd-4dcf-2478-daf8-a2306a879141/e0862929-e242-795d-1e0b-fe7766981a5b/oldWALs 2024-11-23T21:54:46,207 INFO [RS:0;adda16083778:37831 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL adda16083778%2C37831%2C1732398882981:(num 1732398883703) 2024-11-23T21:54:46,208 DEBUG [RS:0;adda16083778:37831 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T21:54:46,208 INFO [RS:0;adda16083778:37831 {}] regionserver.LeaseManager(133): Closed leases 2024-11-23T21:54:46,208 INFO [RS:0;adda16083778:37831 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-23T21:54:46,208 INFO [RS:0;adda16083778:37831 {}] hbase.ChoreService(370): Chore service for: regionserver/adda16083778:0 had [ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-11-23T21:54:46,208 INFO [RS:0;adda16083778:37831 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-23T21:54:46,208 INFO [regionserver/adda16083778:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-23T21:54:46,208 INFO [RS:0;adda16083778:37831 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:37831 2024-11-23T21:54:46,241 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37831-0x1016afab2c80001, quorum=127.0.0.1:51781, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/adda16083778,37831,1732398882981 2024-11-23T21:54:46,241 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33953-0x1016afab2c80000, quorum=127.0.0.1:51781, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-23T21:54:46,241 INFO [RS:0;adda16083778:37831 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-23T21:54:46,250 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [adda16083778,37831,1732398882981] 2024-11-23T21:54:46,258 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/adda16083778,37831,1732398882981 already deleted, retry=false 2024-11-23T21:54:46,258 INFO [RegionServerTracker-0 {}] master.ServerManager(695): Processing expiration of adda16083778,37831,1732398882981 on adda16083778,33953,1732398882834 2024-11-23T21:54:46,266 DEBUG [RegionServerTracker-0 {}] procedure2.ProcedureExecutor(1139): Stored pid=7, state=RUNNABLE:SERVER_CRASH_START, hasLock=false; ServerCrashProcedure adda16083778,37831,1732398882981, splitWal=true, meta=true 2024-11-23T21:54:46,269 INFO [RegionServerTracker-0 {}] assignment.AssignmentManager(1999): Scheduled ServerCrashProcedure pid=7 for adda16083778,37831,1732398882981 (carryingMeta=true) adda16083778,37831,1732398882981/CRASHED/regionCount=2/lock=java.util.concurrent.locks.ReentrantReadWriteLock@1762a3db[Write locks = 1, Read locks = 0], oldState=ONLINE. 2024-11-23T21:54:46,269 INFO [PEWorker-2 {}] procedure.ServerCrashProcedure(169): Start pid=7, state=RUNNABLE:SERVER_CRASH_START, hasLock=true; ServerCrashProcedure adda16083778,37831,1732398882981, splitWal=true, meta=true 2024-11-23T21:54:46,271 INFO [PEWorker-2 {}] procedure.ServerCrashProcedure(339): Splitting WALs pid=7, state=RUNNABLE:SERVER_CRASH_SPLIT_META_LOGS, hasLock=true; ServerCrashProcedure adda16083778,37831,1732398882981, splitWal=true, meta=true, isMeta: true 2024-11-23T21:54:46,273 DEBUG [PEWorker-2 {}] master.MasterWalManager(329): Renamed region directory: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/93be71cd-4dcf-2478-daf8-a2306a879141/e0862929-e242-795d-1e0b-fe7766981a5b/WALs/adda16083778,37831,1732398882981-splitting 2024-11-23T21:54:46,273 INFO [PEWorker-2 {}] master.SplitLogManager(171): file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/93be71cd-4dcf-2478-daf8-a2306a879141/e0862929-e242-795d-1e0b-fe7766981a5b/WALs/adda16083778,37831,1732398882981-splitting dir is empty, no logs to split. 2024-11-23T21:54:46,273 INFO [PEWorker-2 {}] master.SplitWALManager(105): adda16083778,37831,1732398882981 WAL count=0, meta=true 2024-11-23T21:54:46,276 INFO [PEWorker-2 {}] master.SplitLogManager(171): file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/93be71cd-4dcf-2478-daf8-a2306a879141/e0862929-e242-795d-1e0b-fe7766981a5b/WALs/adda16083778,37831,1732398882981-splitting dir is empty, no logs to split. 2024-11-23T21:54:46,276 INFO [PEWorker-2 {}] master.SplitWALManager(105): adda16083778,37831,1732398882981 WAL count=0, meta=true 2024-11-23T21:54:46,276 DEBUG [PEWorker-2 {}] procedure.ServerCrashProcedure(329): Check if adda16083778,37831,1732398882981 WAL splitting is done? wals=0, meta=true 2024-11-23T21:54:46,278 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=8, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-11-23T21:54:46,279 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=8, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-23T21:54:46,279 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(269): Starting pid=8, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-11-23T21:54:46,341 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/adda16083778:0 server-side Connection retries=45 2024-11-23T21:54:46,342 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-23T21:54:46,342 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-23T21:54:46,342 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-23T21:54:46,342 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-23T21:54:46,342 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-23T21:54:46,342 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-23T21:54:46,342 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-23T21:54:46,343 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:34911 2024-11-23T21:54:46,344 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:34911 connecting to ZooKeeper ensemble=127.0.0.1:51781 2024-11-23T21:54:46,345 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-23T21:54:46,345 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-23T21:54:46,350 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37831-0x1016afab2c80001, quorum=127.0.0.1:51781, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-23T21:54:46,350 INFO [RS:0;adda16083778:37831 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-23T21:54:46,350 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37831-0x1016afab2c80001, quorum=127.0.0.1:51781, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-23T21:54:46,350 INFO [RS:0;adda16083778:37831 {}] regionserver.HRegionServer(1031): Exiting; stopping=adda16083778,37831,1732398882981; zookeeper connection closed. 2024-11-23T21:54:46,351 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@6621a974 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@6621a974 2024-11-23T21:54:46,367 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:349110x0, quorum=127.0.0.1:51781, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-23T21:54:46,367 DEBUG [Time-limited test {}] zookeeper.ZKUtil(111): regionserver:34911-0x1016afab2c80003, quorum=127.0.0.1:51781, baseZNode=/hbase Set watcher on existing znode=/hbase/running 2024-11-23T21:54:46,367 DEBUG [pool-298-thread-1 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: INIT 2024-11-23T21:54:46,367 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:34911-0x1016afab2c80003 connected 2024-11-23T21:54:46,368 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-23T21:54:46,369 INFO [Time-limited test {}] bucket.FileIOEngine(99): Allocating cache 400 MB, on the path:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d246ebad-447b-0ccb-984f-a43056e9385e/bucket.cache 2024-11-23T21:54:46,369 INFO [Time-limited test {}] bucket.BucketCache(364): Instantiating BucketCache with acceptableFactor: 0.95, minFactor: 0.85, extraFreeFactor: 0.1, singleFactor: 0.25, multiFactor: 0.5, memoryFactor: 0.25, useStrongRef: false 2024-11-23T21:54:46,369 INFO [Time-limited test {}] bucket.BucketCache(478): Starting BucketCachePersisterThread 2024-11-23T21:54:46,369 INFO [Time-limited test {}] bucket.BucketCachePersister(38): BucketCachePersister started with interval: 100 2024-11-23T21:54:46,369 INFO [Thread-206 {}] bucket.BucketCache(1530): Started retrieving bucket cache from file 2024-11-23T21:54:46,369 INFO [Time-limited test {}] bucket.BucketCache(403): Started bucket cache; ioengine=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d246ebad-447b-0ccb-984f-a43056e9385e/bucket.cache, capacity=400 MB, blockSize=64 KB, writerThreadNum=3, writerQLen=64, persistencePath=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d246ebad-447b-0ccb-984f-a43056e9385e/bucket.persistence, bucketAllocator=org.apache.hadoop.hbase.io.hfile.bucket.BucketAllocator 2024-11-23T21:54:46,370 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-23T21:54:46,370 INFO [Thread-206 {}] bucket.BucketCache(1552): Reading new chunked format of persistence. 2024-11-23T21:54:46,371 DEBUG [Time-limited test {}] zookeeper.ZKUtil(111): regionserver:34911-0x1016afab2c80003, quorum=127.0.0.1:51781, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-11-23T21:54:46,371 INFO [Thread-206 {}] bucket.BucketCache(1705): Retrieved 0 of chunks with blockCount = 0. 2024-11-23T21:54:46,372 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:34911-0x1016afab2c80003, quorum=127.0.0.1:51781, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-23T21:54:46,374 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=34911 2024-11-23T21:54:46,375 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=34911 2024-11-23T21:54:46,375 DEBUG [Thread-206 {}] bucket.BucketCache(1569): Updating region size map after retrieving cached file list 2024-11-23T21:54:46,375 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=34911 2024-11-23T21:54:46,375 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=34911 2024-11-23T21:54:46,375 INFO [Thread-206 {}] bucket.BucketAllocator(343): Cache totalSize=418148352, buckets=199, bucket capacity=2101248=(4*525312)=(FEWEST_ITEMS_IN_BUCKET*(largest configured bucketcache size)) 2024-11-23T21:54:46,375 INFO [Thread-206 {}] bucket.BucketCache(1562): Bucket cache retrieved from file successfully with size: 0 2024-11-23T21:54:46,375 INFO [Thread-206 {}] bucket.BucketCache(413): Persistent bucket cache recovery from /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d246ebad-447b-0ccb-984f-a43056e9385e/bucket.persistence is complete. 2024-11-23T21:54:46,375 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=34911 2024-11-23T21:54:46,377 INFO [RS:2;adda16083778:34911 {}] regionserver.HRegionServer(746): ClusterId : d343274d-4bd1-4b55-b460-a843638d8a30 2024-11-23T21:54:46,377 DEBUG [RS:2;adda16083778:34911 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-23T21:54:46,384 DEBUG [RS:2;adda16083778:34911 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-23T21:54:46,384 DEBUG [RS:2;adda16083778:34911 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-23T21:54:46,393 DEBUG [RS:2;adda16083778:34911 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-23T21:54:46,393 DEBUG [RS:2;adda16083778:34911 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1f676600, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=adda16083778/172.17.0.2:0 2024-11-23T21:54:46,405 DEBUG [RS:2;adda16083778:34911 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:2;adda16083778:34911 2024-11-23T21:54:46,405 INFO [RS:2;adda16083778:34911 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-23T21:54:46,405 INFO [RS:2;adda16083778:34911 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-23T21:54:46,405 DEBUG [RS:2;adda16083778:34911 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-23T21:54:46,406 INFO [RS:2;adda16083778:34911 {}] regionserver.HRegionServer(2659): reportForDuty to master=adda16083778,33953,1732398882834 with port=34911, startcode=1732398886341 2024-11-23T21:54:46,406 DEBUG [RS:2;adda16083778:34911 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-23T21:54:46,408 INFO [HMaster-EventLoopGroup-6-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60253, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=RegionServerStatusService 2024-11-23T21:54:46,408 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=33953 {}] master.ServerManager(363): Checking decommissioned status of RegionServer adda16083778,34911,1732398886341 2024-11-23T21:54:46,408 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=33953 {}] master.ServerManager(517): Registering regionserver=adda16083778,34911,1732398886341 2024-11-23T21:54:46,409 DEBUG [RS:2;adda16083778:34911 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/93be71cd-4dcf-2478-daf8-a2306a879141/e0862929-e242-795d-1e0b-fe7766981a5b 2024-11-23T21:54:46,409 DEBUG [RS:2;adda16083778:34911 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=file:/ 2024-11-23T21:54:46,410 DEBUG [RS:2;adda16083778:34911 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-23T21:54:46,416 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33953-0x1016afab2c80000, quorum=127.0.0.1:51781, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-23T21:54:46,417 DEBUG [RS:2;adda16083778:34911 {}] zookeeper.ZKUtil(111): regionserver:34911-0x1016afab2c80003, quorum=127.0.0.1:51781, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/adda16083778,34911,1732398886341 2024-11-23T21:54:46,417 WARN [RS:2;adda16083778:34911 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-23T21:54:46,417 INFO [RS:2;adda16083778:34911 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-23T21:54:46,417 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [adda16083778,34911,1732398886341] 2024-11-23T21:54:46,417 DEBUG [RS:2;adda16083778:34911 {}] regionserver.HRegionServer(1793): logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/93be71cd-4dcf-2478-daf8-a2306a879141/e0862929-e242-795d-1e0b-fe7766981a5b/WALs/adda16083778,34911,1732398886341 2024-11-23T21:54:46,421 INFO [RS:2;adda16083778:34911 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-23T21:54:46,423 INFO [RS:2;adda16083778:34911 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-23T21:54:46,424 INFO [RS:2;adda16083778:34911 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-23T21:54:46,424 INFO [RS:2;adda16083778:34911 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-23T21:54:46,424 INFO [RS:2;adda16083778:34911 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-23T21:54:46,425 INFO [RS:2;adda16083778:34911 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-23T21:54:46,425 INFO [RS:2;adda16083778:34911 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-23T21:54:46,425 DEBUG [RS:2;adda16083778:34911 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/adda16083778:0, corePoolSize=1, maxPoolSize=1 2024-11-23T21:54:46,425 DEBUG [RS:2;adda16083778:34911 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/adda16083778:0, corePoolSize=1, maxPoolSize=1 2024-11-23T21:54:46,425 DEBUG [RS:2;adda16083778:34911 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/adda16083778:0, corePoolSize=1, maxPoolSize=1 2024-11-23T21:54:46,425 DEBUG [RS:2;adda16083778:34911 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/adda16083778:0, corePoolSize=1, maxPoolSize=1 2024-11-23T21:54:46,425 DEBUG [RS:2;adda16083778:34911 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/adda16083778:0, corePoolSize=1, maxPoolSize=1 2024-11-23T21:54:46,425 DEBUG [RS:2;adda16083778:34911 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/adda16083778:0, corePoolSize=2, maxPoolSize=2 2024-11-23T21:54:46,425 DEBUG [RS:2;adda16083778:34911 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/adda16083778:0, corePoolSize=1, maxPoolSize=1 2024-11-23T21:54:46,425 DEBUG [RS:2;adda16083778:34911 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/adda16083778:0, corePoolSize=1, maxPoolSize=1 2024-11-23T21:54:46,425 DEBUG [RS:2;adda16083778:34911 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/adda16083778:0, corePoolSize=1, maxPoolSize=1 2024-11-23T21:54:46,425 DEBUG [RS:2;adda16083778:34911 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/adda16083778:0, corePoolSize=1, maxPoolSize=1 2024-11-23T21:54:46,425 DEBUG [RS:2;adda16083778:34911 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/adda16083778:0, corePoolSize=1, maxPoolSize=1 2024-11-23T21:54:46,426 DEBUG [RS:2;adda16083778:34911 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/adda16083778:0, corePoolSize=1, maxPoolSize=1 2024-11-23T21:54:46,426 DEBUG [RS:2;adda16083778:34911 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/adda16083778:0, corePoolSize=3, maxPoolSize=3 2024-11-23T21:54:46,426 DEBUG [RS:2;adda16083778:34911 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/adda16083778:0, corePoolSize=3, maxPoolSize=3 2024-11-23T21:54:46,426 INFO [RS:2;adda16083778:34911 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-23T21:54:46,426 INFO [RS:2;adda16083778:34911 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-23T21:54:46,426 INFO [RS:2;adda16083778:34911 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-23T21:54:46,426 INFO [RS:2;adda16083778:34911 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-23T21:54:46,426 INFO [RS:2;adda16083778:34911 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-23T21:54:46,426 INFO [RS:2;adda16083778:34911 {}] hbase.ChoreService(168): Chore ScheduledChore name=adda16083778,34911,1732398886341-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-23T21:54:46,429 DEBUG [adda16083778:33953 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=1, allServersCount=1 2024-11-23T21:54:46,430 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=8 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=adda16083778,45169,1732398883012 2024-11-23T21:54:46,431 INFO [PEWorker-4 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as adda16083778,45169,1732398883012, state=OPENING 2024-11-23T21:54:46,441 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33953-0x1016afab2c80000, quorum=127.0.0.1:51781, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-23T21:54:46,441 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34911-0x1016afab2c80003, quorum=127.0.0.1:51781, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-23T21:54:46,441 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45169-0x1016afab2c80002, quorum=127.0.0.1:51781, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-23T21:54:46,441 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-23T21:54:46,441 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-23T21:54:46,441 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-23T21:54:46,441 DEBUG [PEWorker-4 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=8, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-23T21:54:46,441 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=9, ppid=8, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=adda16083778,45169,1732398883012}] 2024-11-23T21:54:46,445 INFO [RS:2;adda16083778:34911 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-23T21:54:46,445 INFO [RS:2;adda16083778:34911 {}] hbase.ChoreService(168): Chore ScheduledChore name=adda16083778,34911,1732398886341-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-23T21:54:46,445 INFO [RS:2;adda16083778:34911 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-23T21:54:46,445 INFO [RS:2;adda16083778:34911 {}] regionserver.Replication(171): adda16083778,34911,1732398886341 started 2024-11-23T21:54:46,458 INFO [RS:2;adda16083778:34911 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-23T21:54:46,458 INFO [RS:2;adda16083778:34911 {}] regionserver.HRegionServer(1482): Serving as adda16083778,34911,1732398886341, RpcServer on adda16083778/172.17.0.2:34911, sessionid=0x1016afab2c80003 2024-11-23T21:54:46,459 DEBUG [RS:2;adda16083778:34911 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-23T21:54:46,459 DEBUG [RS:2;adda16083778:34911 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager adda16083778,34911,1732398886341 2024-11-23T21:54:46,459 DEBUG [RS:2;adda16083778:34911 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'adda16083778,34911,1732398886341' 2024-11-23T21:54:46,459 DEBUG [RS:2;adda16083778:34911 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-23T21:54:46,460 DEBUG [RS:2;adda16083778:34911 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-23T21:54:46,460 DEBUG [RS:2;adda16083778:34911 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-23T21:54:46,460 DEBUG [RS:2;adda16083778:34911 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-23T21:54:46,460 DEBUG [RS:2;adda16083778:34911 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager adda16083778,34911,1732398886341 2024-11-23T21:54:46,460 DEBUG [RS:2;adda16083778:34911 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'adda16083778,34911,1732398886341' 2024-11-23T21:54:46,460 DEBUG [RS:2;adda16083778:34911 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-23T21:54:46,461 DEBUG [RS:2;adda16083778:34911 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-23T21:54:46,461 DEBUG [RS:2;adda16083778:34911 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-23T21:54:46,461 INFO [RS:2;adda16083778:34911 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-23T21:54:46,462 INFO [RS:2;adda16083778:34911 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-23T21:54:46,563 INFO [RS:2;adda16083778:34911 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-23T21:54:46,564 INFO [RS:2;adda16083778:34911 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=adda16083778%2C34911%2C1732398886341, suffix=, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/93be71cd-4dcf-2478-daf8-a2306a879141/e0862929-e242-795d-1e0b-fe7766981a5b/WALs/adda16083778,34911,1732398886341, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/93be71cd-4dcf-2478-daf8-a2306a879141/e0862929-e242-795d-1e0b-fe7766981a5b/oldWALs, maxLogs=55 2024-11-23T21:54:46,587 INFO [RS:2;adda16083778:34911 {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/93be71cd-4dcf-2478-daf8-a2306a879141/e0862929-e242-795d-1e0b-fe7766981a5b/WALs/adda16083778,34911,1732398886341/adda16083778%2C34911%2C1732398886341.1732398886566 2024-11-23T21:54:46,588 DEBUG [RS:2;adda16083778:34911 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-11-23T21:54:46,594 DEBUG [RSProcedureDispatcher-pool-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-23T21:54:46,595 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-8-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46751, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-23T21:54:46,597 INFO [RS_OPEN_META-regionserver/adda16083778:0-0 {event_type=M_RS_OPEN_META, pid=9}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-11-23T21:54:46,597 INFO [RS_OPEN_META-regionserver/adda16083778:0-0 {event_type=M_RS_OPEN_META, pid=9}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-23T21:54:46,598 INFO [RS_OPEN_META-regionserver/adda16083778:0-0 {event_type=M_RS_OPEN_META, pid=9}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-11-23T21:54:46,599 INFO [RS_OPEN_META-regionserver/adda16083778:0-0 {event_type=M_RS_OPEN_META, pid=9}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=adda16083778%2C45169%2C1732398883012.meta, suffix=.meta, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/93be71cd-4dcf-2478-daf8-a2306a879141/e0862929-e242-795d-1e0b-fe7766981a5b/WALs/adda16083778,45169,1732398883012, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/93be71cd-4dcf-2478-daf8-a2306a879141/e0862929-e242-795d-1e0b-fe7766981a5b/oldWALs, maxLogs=55 2024-11-23T21:54:46,621 INFO [RS_OPEN_META-regionserver/adda16083778:0-0 {event_type=M_RS_OPEN_META, pid=9}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/93be71cd-4dcf-2478-daf8-a2306a879141/e0862929-e242-795d-1e0b-fe7766981a5b/WALs/adda16083778,45169,1732398883012/adda16083778%2C45169%2C1732398883012.meta.1732398886600.meta 2024-11-23T21:54:46,622 DEBUG [RS_OPEN_META-regionserver/adda16083778:0-0 {event_type=M_RS_OPEN_META, pid=9}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-11-23T21:54:46,622 DEBUG [RS_OPEN_META-regionserver/adda16083778:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-11-23T21:54:46,622 DEBUG [RS_OPEN_META-regionserver/adda16083778:0-0 {event_type=M_RS_OPEN_META, pid=9}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-11-23T21:54:46,622 DEBUG [RS_OPEN_META-regionserver/adda16083778:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-11-23T21:54:46,622 INFO [RS_OPEN_META-regionserver/adda16083778:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-11-23T21:54:46,622 DEBUG [RS_OPEN_META-regionserver/adda16083778:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-11-23T21:54:46,623 DEBUG [RS_OPEN_META-regionserver/adda16083778:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-23T21:54:46,623 DEBUG [RS_OPEN_META-regionserver/adda16083778:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-11-23T21:54:46,623 DEBUG [RS_OPEN_META-regionserver/adda16083778:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-11-23T21:54:46,623 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family info of region 1588230740 2024-11-23T21:54:46,624 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-23T21:54:46,624 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T21:54:46,625 DEBUG [StoreFileOpener-1588230740-info-1 {}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/93be71cd-4dcf-2478-daf8-a2306a879141/e0862929-e242-795d-1e0b-fe7766981a5b/data/hbase/meta/1588230740/info/1b70fd2dcd8f449b823e9ebf60be2907, delay=923 ms 2024-11-23T21:54:46,626 DEBUG [StoreOpener-1588230740-1 {}] regionserver.StoreEngine(278): loaded file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/93be71cd-4dcf-2478-daf8-a2306a879141/e0862929-e242-795d-1e0b-fe7766981a5b/data/hbase/meta/1588230740/info/1b70fd2dcd8f449b823e9ebf60be2907 2024-11-23T21:54:46,626 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-23T21:54:46,626 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family ns of region 1588230740 2024-11-23T21:54:46,627 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-23T21:54:46,627 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T21:54:46,628 DEBUG [StoreFileOpener-1588230740-ns-1 {}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/93be71cd-4dcf-2478-daf8-a2306a879141/e0862929-e242-795d-1e0b-fe7766981a5b/data/hbase/meta/1588230740/ns/cfd1bd3e99364567803dec829e99ee1a, delay=900 ms 2024-11-23T21:54:46,630 DEBUG [StoreOpener-1588230740-1 {}] regionserver.StoreEngine(278): loaded file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/93be71cd-4dcf-2478-daf8-a2306a879141/e0862929-e242-795d-1e0b-fe7766981a5b/data/hbase/meta/1588230740/ns/cfd1bd3e99364567803dec829e99ee1a 2024-11-23T21:54:46,630 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-23T21:54:46,630 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family rep_barrier of region 1588230740 2024-11-23T21:54:46,631 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-23T21:54:46,631 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T21:54:46,631 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-23T21:54:46,631 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family table of region 1588230740 2024-11-23T21:54:46,631 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-23T21:54:46,632 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T21:54:46,633 DEBUG [StoreFileOpener-1588230740-table-1 {}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/93be71cd-4dcf-2478-daf8-a2306a879141/e0862929-e242-795d-1e0b-fe7766981a5b/data/hbase/meta/1588230740/table/74a27669f3a94e7cbe5394a8064aaee3, delay=900 ms 2024-11-23T21:54:46,634 DEBUG [StoreOpener-1588230740-1 {}] regionserver.StoreEngine(278): loaded file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/93be71cd-4dcf-2478-daf8-a2306a879141/e0862929-e242-795d-1e0b-fe7766981a5b/data/hbase/meta/1588230740/table/74a27669f3a94e7cbe5394a8064aaee3 2024-11-23T21:54:46,634 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-23T21:54:46,634 DEBUG [RS_OPEN_META-regionserver/adda16083778:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-23T21:54:46,635 DEBUG [RS_OPEN_META-regionserver/adda16083778:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/93be71cd-4dcf-2478-daf8-a2306a879141/e0862929-e242-795d-1e0b-fe7766981a5b/data/hbase/meta/1588230740 2024-11-23T21:54:46,635 DEBUG [RS_OPEN_META-regionserver/adda16083778:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/93be71cd-4dcf-2478-daf8-a2306a879141/e0862929-e242-795d-1e0b-fe7766981a5b/data/hbase/meta/1588230740 2024-11-23T21:54:46,636 DEBUG [RS_OPEN_META-regionserver/adda16083778:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-23T21:54:46,636 DEBUG [RS_OPEN_META-regionserver/adda16083778:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-23T21:54:46,636 DEBUG [RS_OPEN_META-regionserver/adda16083778:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-23T21:54:46,637 DEBUG [RS_OPEN_META-regionserver/adda16083778:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-23T21:54:46,638 INFO [RS_OPEN_META-regionserver/adda16083778:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=15; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=60098540, jitterRate=-0.10446196794509888}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-23T21:54:46,638 DEBUG [RS_OPEN_META-regionserver/adda16083778:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-11-23T21:54:46,639 DEBUG [RS_OPEN_META-regionserver/adda16083778:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1732398886623Writing region info on filesystem at 1732398886623Initializing all the Stores at 1732398886623Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732398886623Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732398886623Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732398886623Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732398886623Cleaning up temporary data from old regions at 1732398886636 (+13 ms)Running coprocessor post-open hooks at 1732398886638 (+2 ms)Region opened successfully at 1732398886639 (+1 ms) 2024-11-23T21:54:46,640 INFO [RS_OPEN_META-regionserver/adda16083778:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=9, masterSystemTime=1732398886593 2024-11-23T21:54:46,641 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta 2024-11-23T21:54:46,641 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_meta Metrics about Tables on a single HBase RegionServer 2024-11-23T21:54:46,642 DEBUG [RS_OPEN_META-regionserver/adda16083778:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-11-23T21:54:46,643 INFO [RS_OPEN_META-regionserver/adda16083778:0-0 {event_type=M_RS_OPEN_META, pid=9}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-11-23T21:54:46,643 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=8 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=15, regionLocation=adda16083778,45169,1732398883012 2024-11-23T21:54:46,643 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-23T21:54:46,643 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint Metrics about HBase RegionObservers 2024-11-23T21:54:46,643 INFO [PEWorker-1 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as adda16083778,45169,1732398883012, state=OPEN 2024-11-23T21:54:46,644 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_master_table_store 2024-11-23T21:54:46,644 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_master_table_store Metrics about Tables on a single HBase RegionServer 2024-11-23T21:54:46,649 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=33953 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=adda16083778,45169,1732398883012, table=hbase:meta, region=1588230740. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-23T21:54:46,666 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33953-0x1016afab2c80000, quorum=127.0.0.1:51781, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-23T21:54:46,666 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45169-0x1016afab2c80002, quorum=127.0.0.1:51781, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-23T21:54:46,666 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34911-0x1016afab2c80003, quorum=127.0.0.1:51781, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-23T21:54:46,666 DEBUG [PEWorker-1 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=9, ppid=8, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=adda16083778,45169,1732398883012 2024-11-23T21:54:46,666 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-23T21:54:46,666 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-23T21:54:46,666 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-23T21:54:46,668 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=9, resume processing ppid=8 2024-11-23T21:54:46,669 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=9, ppid=8, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=adda16083778,45169,1732398883012 in 225 msec 2024-11-23T21:54:46,671 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=8, resume processing ppid=7 2024-11-23T21:54:46,671 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=8, ppid=7, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 392 msec 2024-11-23T21:54:46,671 INFO [PEWorker-3 {}] procedure.ServerCrashProcedure(207): adda16083778,37831,1732398882981 had 2 regions 2024-11-23T21:54:46,672 INFO [PEWorker-3 {}] procedure.ServerCrashProcedure(339): Splitting WALs pid=7, state=RUNNABLE:SERVER_CRASH_SPLIT_LOGS, hasLock=true; ServerCrashProcedure adda16083778,37831,1732398882981, splitWal=true, meta=true, isMeta: false 2024-11-23T21:54:46,673 INFO [PEWorker-3 {}] master.SplitLogManager(171): file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/93be71cd-4dcf-2478-daf8-a2306a879141/e0862929-e242-795d-1e0b-fe7766981a5b/WALs/adda16083778,37831,1732398882981-splitting dir is empty, no logs to split. 2024-11-23T21:54:46,673 INFO [PEWorker-3 {}] master.SplitWALManager(105): adda16083778,37831,1732398882981 WAL count=0, meta=false 2024-11-23T21:54:46,674 INFO [PEWorker-3 {}] master.SplitLogManager(171): file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/93be71cd-4dcf-2478-daf8-a2306a879141/e0862929-e242-795d-1e0b-fe7766981a5b/WALs/adda16083778,37831,1732398882981-splitting dir is empty, no logs to split. 2024-11-23T21:54:46,674 INFO [PEWorker-3 {}] master.SplitWALManager(105): adda16083778,37831,1732398882981 WAL count=0, meta=false 2024-11-23T21:54:46,674 DEBUG [PEWorker-3 {}] procedure.ServerCrashProcedure(329): Check if adda16083778,37831,1732398882981 WAL splitting is done? wals=0, meta=false 2024-11-23T21:54:46,675 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=10, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1732398884153, region=d1d01dcb0a94d6c09a6bd0b1d689b7ea, ASSIGN}] 2024-11-23T21:54:46,677 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=10, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1732398884153, region=d1d01dcb0a94d6c09a6bd0b1d689b7ea, ASSIGN 2024-11-23T21:54:46,677 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=10, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1732398884153, region=d1d01dcb0a94d6c09a6bd0b1d689b7ea, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-11-23T21:54:46,828 DEBUG [adda16083778:33953 {}] balancer.BalancerClusterState(204): Hosts are {adda16083778=0} racks are {/default-rack=0} 2024-11-23T21:54:46,828 DEBUG [adda16083778:33953 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-11-23T21:54:46,829 DEBUG [adda16083778:33953 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-11-23T21:54:46,829 DEBUG [adda16083778:33953 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-11-23T21:54:46,829 DEBUG [adda16083778:33953 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-11-23T21:54:46,829 INFO [adda16083778:33953 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-11-23T21:54:46,829 INFO [adda16083778:33953 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-11-23T21:54:46,829 DEBUG [adda16083778:33953 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-11-23T21:54:46,830 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=10 updating hbase:meta row=d1d01dcb0a94d6c09a6bd0b1d689b7ea, regionState=OPENING, regionLocation=adda16083778,45169,1732398883012 2024-11-23T21:54:46,832 WARN [MiniHBaseClusterRegionServer-EventLoopGroup-8-3 {}] ipc.NettyRpcConnection$2(409): Exception encountered while connecting to the server adda16083778:37831 org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: adda16083778/172.17.0.2:37831 Caused by: java.net.ConnectException: finishConnect(..) failed: Connection refused at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.newConnectException0(Errors.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.handleConnectErrno(Errors.java:131) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Socket.finishConnect(Socket.java:359) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.doFinishConnect(AbstractEpollChannel.java:715) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:692) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T21:54:46,836 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-8-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=adda16083778,37831,1732398882981, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=adda16083778,37831,1732398882981, seqNum=-1, error=java.net.ConnectException: Call to address=adda16083778:37831 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: adda16083778/172.17.0.2:37831 2024-11-23T21:54:46,837 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-8-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:meta,,1.1588230740, hostname=adda16083778,37831,1732398882981, seqNum=-1 is java.net.ConnectException: finishConnect(..) failed: Connection refused 2024-11-23T21:54:46,838 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-8-3 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:meta,,1.1588230740, hostname=adda16083778,37831,1732398882981, seqNum=-1 from cache 2024-11-23T21:54:46,839 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-8-3 {}] ipc.FailedServers(52): Added failed server with address adda16083778:37831 to list caused by org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: adda16083778/172.17.0.2:37831 2024-11-23T21:54:46,946 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-23T21:54:46,946 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=adda16083778,45169,1732398883012, seqNum=-1] 2024-11-23T21:54:46,947 DEBUG [Async-Client-Retry-Timer-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-23T21:54:46,948 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-8-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36913, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-23T21:54:46,951 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-8-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=10, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1732398884153, region=d1d01dcb0a94d6c09a6bd0b1d689b7ea, ASSIGN because future has completed 2024-11-23T21:54:46,951 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=11, ppid=10, state=RUNNABLE, hasLock=false; OpenRegionProcedure d1d01dcb0a94d6c09a6bd0b1d689b7ea, server=adda16083778,45169,1732398883012}] 2024-11-23T21:54:46,959 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster 2024-11-23T21:54:46,959 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-23T21:54:46,959 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79) at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611) at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.io.hfile.TestBlockEvictionOnRegionMovement.tearDown(TestBlockEvictionOnRegionMovement.java:174) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-23T21:54:46,959 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T21:54:46,959 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T21:54:46,959 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-23T21:54:46,960 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2024-11-23T21:54:46,960 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=1485292123, stopped=false 2024-11-23T21:54:46,960 INFO [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=adda16083778,33953,1732398882834 2024-11-23T21:54:46,975 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45169-0x1016afab2c80002, quorum=127.0.0.1:51781, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-23T21:54:46,975 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33953-0x1016afab2c80000, quorum=127.0.0.1:51781, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-23T21:54:46,975 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34911-0x1016afab2c80003, quorum=127.0.0.1:51781, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-23T21:54:46,975 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45169-0x1016afab2c80002, quorum=127.0.0.1:51781, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-23T21:54:46,975 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33953-0x1016afab2c80000, quorum=127.0.0.1:51781, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-23T21:54:46,975 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34911-0x1016afab2c80003, quorum=127.0.0.1:51781, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-23T21:54:46,975 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-23T21:54:46,975 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-23T21:54:46,976 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277) at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265) at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416) at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.io.hfile.TestBlockEvictionOnRegionMovement.tearDown(TestBlockEvictionOnRegionMovement.java:174) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-23T21:54:46,976 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:34911-0x1016afab2c80003, quorum=127.0.0.1:51781, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-23T21:54:46,976 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:45169-0x1016afab2c80002, quorum=127.0.0.1:51781, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-23T21:54:46,976 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T21:54:46,976 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:33953-0x1016afab2c80000, quorum=127.0.0.1:51781, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-23T21:54:46,977 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server 'adda16083778,45169,1732398883012' ***** 2024-11-23T21:54:46,977 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-11-23T21:54:46,977 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server 'adda16083778,34911,1732398886341' ***** 2024-11-23T21:54:46,977 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-11-23T21:54:46,977 INFO [RS:1;adda16083778:45169 {}] bucket.BucketCache(1738): Disabling cache 2024-11-23T21:54:46,978 INFO [RS:2;adda16083778:34911 {}] bucket.BucketCache(1738): Disabling cache 2024-11-23T21:54:46,978 INFO [RS:1;adda16083778:45169 {}] bucket.BucketCache(1762): Shutdown bucket cache: IO persistent=true; path to write=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d246ebad-447b-0ccb-984f-a43056e9385e/bucket.persistence 2024-11-23T21:54:46,978 INFO [Time-limited test-BucketCacheWriter-0 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-0 exiting, cacheEnabled=false 2024-11-23T21:54:46,978 INFO [Time-limited test-BucketCacheWriter-2 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-2 exiting, cacheEnabled=false 2024-11-23T21:54:46,978 INFO [Time-limited test-BucketCacheWriter-1 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-1 exiting, cacheEnabled=false 2024-11-23T21:54:46,978 INFO [RS:1;adda16083778:45169 {}] bucket.BucketCache(1768): Shutting down cache persister thread. 2024-11-23T21:54:46,978 INFO [RS:2;adda16083778:34911 {}] bucket.BucketCache(1762): Shutdown bucket cache: IO persistent=true; path to write=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d246ebad-447b-0ccb-984f-a43056e9385e/bucket.persistence 2024-11-23T21:54:46,978 INFO [Time-limited test-BucketCacheWriter-0 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-0 exiting, cacheEnabled=false 2024-11-23T21:54:46,978 INFO [Time-limited test-BucketCacheWriter-1 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-1 exiting, cacheEnabled=false 2024-11-23T21:54:46,978 WARN [bucket-cache-persister {}] bucket.BucketCachePersister(64): Interrupting BucketCachePersister thread. java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCachePersister.run(BucketCachePersister.java:45) ~[classes/:?] 2024-11-23T21:54:46,979 INFO [Time-limited test-BucketCacheWriter-2 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-2 exiting, cacheEnabled=false 2024-11-23T21:54:46,979 INFO [RS:2;adda16083778:34911 {}] bucket.BucketCache(1768): Shutting down cache persister thread. 2024-11-23T21:54:46,979 DEBUG [RS:1;adda16083778:45169 {}] bucket.BucketCache(1487): Thread RS:1;adda16083778:45169 started persisting bucket cache to file 2024-11-23T21:54:46,979 WARN [bucket-cache-persister {}] bucket.BucketCachePersister(64): Interrupting BucketCachePersister thread. java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCachePersister.run(BucketCachePersister.java:45) ~[classes/:?] 2024-11-23T21:54:46,979 DEBUG [RS:1;adda16083778:45169 {}] bucket.BucketCache(1494): Persist in new chunked persistence format. 2024-11-23T21:54:46,979 DEBUG [RS:1;adda16083778:45169 {}] bucket.BucketCache(1673): persistToFile: before persisting backing map size: 0, fullycachedFiles size: 0, chunkSize: 10000 2024-11-23T21:54:46,983 DEBUG [RS:1;adda16083778:45169 {}] bucket.BucketCache(1680): persistToFile: after persisting backing map size: 0, fullycachedFiles size: 0 2024-11-23T21:54:46,983 DEBUG [RS:1;adda16083778:45169 {}] bucket.BucketCache(1498): PersistToFile: after persisting backing map size: 0, fullycachedFiles size: 0, file name: bucket.persistence1732398886979 2024-11-23T21:54:46,983 DEBUG [RS:1;adda16083778:45169 {}] bucket.BucketCache(1509): Thread RS:1;adda16083778:45169 finished persisting bucket cache to file, renaming 2024-11-23T21:54:46,984 INFO [RS:1;adda16083778:45169 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-23T21:54:46,984 INFO [RS:1;adda16083778:45169 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-23T21:54:46,984 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-23T21:54:46,984 INFO [RS:1;adda16083778:45169 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-23T21:54:46,984 INFO [RS:1;adda16083778:45169 {}] regionserver.HRegionServer(959): stopping server adda16083778,45169,1732398883012 2024-11-23T21:54:46,984 INFO [RS:1;adda16083778:45169 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-23T21:54:46,984 INFO [RS:1;adda16083778:45169 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:1;adda16083778:45169. 2024-11-23T21:54:46,984 DEBUG [RS:1;adda16083778:45169 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-23T21:54:46,984 DEBUG [RS:1;adda16083778:45169 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T21:54:46,984 INFO [RS:1;adda16083778:45169 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-23T21:54:46,984 INFO [RS:1;adda16083778:45169 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-23T21:54:46,984 INFO [RS:1;adda16083778:45169 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-23T21:54:46,985 INFO [RS:1;adda16083778:45169 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-11-23T21:54:46,985 INFO [RS:1;adda16083778:45169 {}] regionserver.HRegionServer(1321): Waiting on 1 regions to close 2024-11-23T21:54:46,985 DEBUG [RS:1;adda16083778:45169 {}] regionserver.HRegionServer(1325): Online Regions={1588230740=hbase:meta,,1.1588230740} 2024-11-23T21:54:46,985 DEBUG [RS:1;adda16083778:45169 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-23T21:54:46,985 INFO [RS_CLOSE_META-regionserver/adda16083778:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1662): Closing region hbase:meta,,1.1588230740 during a graceful stop, and cache persistence is on, so setting evict on close to false. 2024-11-23T21:54:46,985 DEBUG [RS_CLOSE_META-regionserver/adda16083778:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-23T21:54:46,985 INFO [RS_CLOSE_META-regionserver/adda16083778:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-23T21:54:46,985 DEBUG [RS_CLOSE_META-regionserver/adda16083778:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-23T21:54:46,985 DEBUG [RS_CLOSE_META-regionserver/adda16083778:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-23T21:54:46,985 DEBUG [RS_CLOSE_META-regionserver/adda16083778:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-23T21:54:46,986 INFO [RS_CLOSE_META-regionserver/adda16083778:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=485 B heapSize=1.70 KB 2024-11-23T21:54:46,989 DEBUG [RS:2;adda16083778:34911 {}] bucket.BucketCache(1487): Thread RS:2;adda16083778:34911 started persisting bucket cache to file 2024-11-23T21:54:46,989 DEBUG [RS:2;adda16083778:34911 {}] bucket.BucketCache(1494): Persist in new chunked persistence format. 2024-11-23T21:54:46,989 DEBUG [RS:2;adda16083778:34911 {}] bucket.BucketCache(1673): persistToFile: before persisting backing map size: 0, fullycachedFiles size: 0, chunkSize: 10000 2024-11-23T21:54:46,992 DEBUG [RS:2;adda16083778:34911 {}] bucket.BucketCache(1680): persistToFile: after persisting backing map size: 0, fullycachedFiles size: 0 2024-11-23T21:54:46,992 DEBUG [RS:2;adda16083778:34911 {}] bucket.BucketCache(1498): PersistToFile: after persisting backing map size: 0, fullycachedFiles size: 0, file name: bucket.persistence1732398886989 2024-11-23T21:54:46,992 DEBUG [RS:2;adda16083778:34911 {}] bucket.BucketCache(1509): Thread RS:2;adda16083778:34911 finished persisting bucket cache to file, renaming 2024-11-23T21:54:46,993 DEBUG [RS_CLOSE_META-regionserver/adda16083778:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFile$WriterFactory(308): Unable to set drop behind on da103ba3f0fa42dc9d908eeeb85ec186 2024-11-23T21:54:47,006 DEBUG [RS_CLOSE_META-regionserver/adda16083778:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/93be71cd-4dcf-2478-daf8-a2306a879141/e0862929-e242-795d-1e0b-fe7766981a5b/data/hbase/meta/1588230740/.tmp/info/da103ba3f0fa42dc9d908eeeb85ec186 is 209, key is testBlockEvictionOnGracefulStop1732398884153,,1732398884158.d1d01dcb0a94d6c09a6bd0b1d689b7ea./info:regioninfo/1732398886830/Put/seqid=0 2024-11-23T21:54:47,006 INFO [RS_CLOSE_META-regionserver/adda16083778:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=485 B at sequenceid=18 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/93be71cd-4dcf-2478-daf8-a2306a879141/e0862929-e242-795d-1e0b-fe7766981a5b/data/hbase/meta/1588230740/.tmp/info/da103ba3f0fa42dc9d908eeeb85ec186 2024-11-23T21:54:47,008 DEBUG [RS_CLOSE_META-regionserver/adda16083778:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/93be71cd-4dcf-2478-daf8-a2306a879141/e0862929-e242-795d-1e0b-fe7766981a5b/data/hbase/meta/1588230740/.tmp/info/da103ba3f0fa42dc9d908eeeb85ec186 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/93be71cd-4dcf-2478-daf8-a2306a879141/e0862929-e242-795d-1e0b-fe7766981a5b/data/hbase/meta/1588230740/info/da103ba3f0fa42dc9d908eeeb85ec186 2024-11-23T21:54:47,009 DEBUG [RS_CLOSE_META-regionserver/adda16083778:0-0 {event_type=M_RS_CLOSE_META}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/93be71cd-4dcf-2478-daf8-a2306a879141/e0862929-e242-795d-1e0b-fe7766981a5b/data/hbase/meta/1588230740/info/da103ba3f0fa42dc9d908eeeb85ec186, delay=987 ms 2024-11-23T21:54:47,011 INFO [RS_CLOSE_META-regionserver/adda16083778:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/93be71cd-4dcf-2478-daf8-a2306a879141/e0862929-e242-795d-1e0b-fe7766981a5b/data/hbase/meta/1588230740/info/da103ba3f0fa42dc9d908eeeb85ec186, entries=3, sequenceid=18, filesize=5.9 K 2024-11-23T21:54:47,011 INFO [RS_CLOSE_META-regionserver/adda16083778:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~485 B/485, heapSize ~952 B/952, currentSize=0 B/0 for 1588230740 in 26ms, sequenceid=18, compaction requested=false 2024-11-23T21:54:47,012 DEBUG [StoreFileCloser-1588230740-info-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/93be71cd-4dcf-2478-daf8-a2306a879141/e0862929-e242-795d-1e0b-fe7766981a5b/data/hbase/meta/1588230740/info/1b70fd2dcd8f449b823e9ebf60be2907 2024-11-23T21:54:47,012 DEBUG [StoreFileCloser-1588230740-info-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/93be71cd-4dcf-2478-daf8-a2306a879141/e0862929-e242-795d-1e0b-fe7766981a5b/data/hbase/meta/1588230740/info/da103ba3f0fa42dc9d908eeeb85ec186 2024-11-23T21:54:47,013 DEBUG [StoreFileCloser-1588230740-ns-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/93be71cd-4dcf-2478-daf8-a2306a879141/e0862929-e242-795d-1e0b-fe7766981a5b/data/hbase/meta/1588230740/ns/cfd1bd3e99364567803dec829e99ee1a 2024-11-23T21:54:47,014 DEBUG [StoreFileCloser-1588230740-table-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/93be71cd-4dcf-2478-daf8-a2306a879141/e0862929-e242-795d-1e0b-fe7766981a5b/data/hbase/meta/1588230740/table/74a27669f3a94e7cbe5394a8064aaee3 2024-11-23T21:54:47,019 INFO [RS:2;adda16083778:34911 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-23T21:54:47,019 INFO [RS:2;adda16083778:34911 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-23T21:54:47,019 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-23T21:54:47,019 INFO [RS:2;adda16083778:34911 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-23T21:54:47,019 INFO [RS:2;adda16083778:34911 {}] regionserver.HRegionServer(959): stopping server adda16083778,34911,1732398886341 2024-11-23T21:54:47,019 INFO [RS:2;adda16083778:34911 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-23T21:54:47,019 INFO [RS:2;adda16083778:34911 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:2;adda16083778:34911. 2024-11-23T21:54:47,019 DEBUG [RS:2;adda16083778:34911 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-23T21:54:47,019 DEBUG [RS:2;adda16083778:34911 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T21:54:47,019 INFO [RS:2;adda16083778:34911 {}] regionserver.HRegionServer(976): stopping server adda16083778,34911,1732398886341; all regions closed. 2024-11-23T21:54:47,020 DEBUG [RS_CLOSE_META-regionserver/adda16083778:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/93be71cd-4dcf-2478-daf8-a2306a879141/e0862929-e242-795d-1e0b-fe7766981a5b/data/hbase/meta/1588230740/recovered.edits/21.seqid, newMaxSeqId=21, maxSeqId=14 2024-11-23T21:54:47,021 DEBUG [RS_CLOSE_META-regionserver/adda16083778:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-23T21:54:47,021 INFO [RS_CLOSE_META-regionserver/adda16083778:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-23T21:54:47,021 DEBUG [RS_CLOSE_META-regionserver/adda16083778:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1732398886985Running coprocessor pre-close hooks at 1732398886985Disabling compacts and flushes for region at 1732398886985Disabling writes for close at 1732398886985Obtaining lock to block concurrent updates at 1732398886986 (+1 ms)Preparing flush snapshotting stores in 1588230740 at 1732398886986Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=485, getHeapSize=1672, getOffHeapSize=0, getCellsCount=3 at 1732398886986Flushing stores of hbase:meta,,1.1588230740 at 1732398886986Flushing 1588230740/info: creating writer at 1732398886986Flushing 1588230740/info: appending metadata at 1732398887005 (+19 ms)Flushing 1588230740/info: closing flushed file at 1732398887005Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@403a0858: reopening flushed file at 1732398887008 (+3 ms)Finished flush of dataSize ~485 B/485, heapSize ~952 B/952, currentSize=0 B/0 for 1588230740 in 26ms, sequenceid=18, compaction requested=false at 1732398887011 (+3 ms)Writing region close event to WAL at 1732398887014 (+3 ms)Running coprocessor post-close hooks at 1732398887021 (+7 ms)Closed at 1732398887021 2024-11-23T21:54:47,021 DEBUG [RS:2;adda16083778:34911 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/93be71cd-4dcf-2478-daf8-a2306a879141/e0862929-e242-795d-1e0b-fe7766981a5b/oldWALs 2024-11-23T21:54:47,021 DEBUG [RS_CLOSE_META-regionserver/adda16083778:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-11-23T21:54:47,021 INFO [RS:2;adda16083778:34911 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL adda16083778%2C34911%2C1732398886341:(num 1732398886566) 2024-11-23T21:54:47,021 DEBUG [RS:2;adda16083778:34911 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T21:54:47,021 INFO [RS:2;adda16083778:34911 {}] regionserver.LeaseManager(133): Closed leases 2024-11-23T21:54:47,021 INFO [RS:2;adda16083778:34911 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-23T21:54:47,022 INFO [RS:2;adda16083778:34911 {}] hbase.ChoreService(370): Chore service for: regionserver/adda16083778:0 had [ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS] on shutdown 2024-11-23T21:54:47,022 INFO [RS:2;adda16083778:34911 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-23T21:54:47,022 INFO [regionserver/adda16083778:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-23T21:54:47,022 INFO [RS:2;adda16083778:34911 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-23T21:54:47,022 INFO [RS:2;adda16083778:34911 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-23T21:54:47,022 INFO [RS:2;adda16083778:34911 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-23T21:54:47,022 INFO [RS:2;adda16083778:34911 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:34911 2024-11-23T21:54:47,027 INFO [regionserver/adda16083778:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-23T21:54:47,049 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33953-0x1016afab2c80000, quorum=127.0.0.1:51781, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-23T21:54:47,049 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34911-0x1016afab2c80003, quorum=127.0.0.1:51781, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/adda16083778,34911,1732398886341 2024-11-23T21:54:47,049 INFO [RS:2;adda16083778:34911 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-23T21:54:47,050 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [adda16083778,34911,1732398886341] 2024-11-23T21:54:47,065 INFO [regionserver/adda16083778:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-23T21:54:47,066 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/adda16083778,34911,1732398886341 already deleted, retry=false 2024-11-23T21:54:47,066 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; adda16083778,34911,1732398886341 expired; onlineServers=1 2024-11-23T21:54:47,104 DEBUG [RSProcedureDispatcher-pool-0 {}] procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall(271): Request to adda16083778,45169,1732398883012 failed, try=0 org.apache.hadoop.hbase.ipc.StoppedRpcClientException: Call to address=adda16083778:45169 failed on local exception: org.apache.hadoop.hbase.ipc.StoppedRpcClientException at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:163) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.sendRequest(RSProcedureDispatcher.java:431) ~[classes/:?] at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.run(RSProcedureDispatcher.java:398) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] at --------Future.get--------(Unknown Source) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:237) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:395) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:451) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$RpcChannelImplementation.callMethod(AbstractRpcClient.java:628) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos$AdminService$Stub.executeProcedures(AdminProtos.java:35059) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRegionServerAdmin.lambda$executeProcedures$20(AsyncRegionServerAdmin.java:218) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRegionServerAdmin.call(AsyncRegionServerAdmin.java:103) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRegionServerAdmin.call(AsyncRegionServerAdmin.java:121) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRegionServerAdmin.executeProcedures(AsyncRegionServerAdmin.java:218) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.sendRequest(RSProcedureDispatcher.java:431) ~[classes/:?] at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.run(RSProcedureDispatcher.java:398) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.StoppedRpcClientException at org.apache.hadoop.hbase.ipc.AbstractRpcClient.getConnection(AbstractRpcClient.java:366) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:448) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 11 more 2024-11-23T21:54:47,106 WARN [RSProcedureDispatcher-pool-0 {}] procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall(288): adda16083778,45169,1732398883012 tells us DoNotRetry due to org.apache.hadoop.hbase.ipc.StoppedRpcClientException: Call to address=adda16083778:45169 failed on local exception: org.apache.hadoop.hbase.ipc.StoppedRpcClientException, try=0, give up 2024-11-23T21:54:47,106 WARN [RSProcedureDispatcher-pool-0 {}] assignment.RegionRemoteProcedureBase(139): The remote operation pid=11, ppid=10, state=RUNNABLE, hasLock=false; OpenRegionProcedure d1d01dcb0a94d6c09a6bd0b1d689b7ea, server=adda16083778,45169,1732398883012 for region state=OPENING, location=adda16083778,45169,1732398883012, table=testBlockEvictionOnGracefulStop1732398884153, region=d1d01dcb0a94d6c09a6bd0b1d689b7ea to server adda16083778,45169,1732398883012 failed org.apache.hadoop.hbase.ipc.StoppedRpcClientException: Call to address=adda16083778:45169 failed on local exception: org.apache.hadoop.hbase.ipc.StoppedRpcClientException at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:163) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.sendRequest(RSProcedureDispatcher.java:431) ~[classes/:?] at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.run(RSProcedureDispatcher.java:398) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] at --------Future.get--------(Unknown Source) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:237) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:395) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:451) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$RpcChannelImplementation.callMethod(AbstractRpcClient.java:628) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos$AdminService$Stub.executeProcedures(AdminProtos.java:35059) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRegionServerAdmin.lambda$executeProcedures$20(AsyncRegionServerAdmin.java:218) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRegionServerAdmin.call(AsyncRegionServerAdmin.java:103) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRegionServerAdmin.call(AsyncRegionServerAdmin.java:121) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRegionServerAdmin.executeProcedures(AsyncRegionServerAdmin.java:218) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.sendRequest(RSProcedureDispatcher.java:431) ~[classes/:?] at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.run(RSProcedureDispatcher.java:398) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.StoppedRpcClientException at org.apache.hadoop.hbase.ipc.AbstractRpcClient.getConnection(AbstractRpcClient.java:366) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:448) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 11 more 2024-11-23T21:54:47,158 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34911-0x1016afab2c80003, quorum=127.0.0.1:51781, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-23T21:54:47,158 INFO [RS:2;adda16083778:34911 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-23T21:54:47,159 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34911-0x1016afab2c80003, quorum=127.0.0.1:51781, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-23T21:54:47,159 INFO [RS:2;adda16083778:34911 {}] regionserver.HRegionServer(1031): Exiting; stopping=adda16083778,34911,1732398886341; zookeeper connection closed. 2024-11-23T21:54:47,159 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@a5f1bba {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@a5f1bba 2024-11-23T21:54:47,185 INFO [RS:1;adda16083778:45169 {}] regionserver.HRegionServer(976): stopping server adda16083778,45169,1732398883012; all regions closed. 2024-11-23T21:54:47,192 DEBUG [RS:1;adda16083778:45169 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/93be71cd-4dcf-2478-daf8-a2306a879141/e0862929-e242-795d-1e0b-fe7766981a5b/oldWALs 2024-11-23T21:54:47,192 INFO [RS:1;adda16083778:45169 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL adda16083778%2C45169%2C1732398883012.meta:.meta(num 1732398886600) 2024-11-23T21:54:47,196 DEBUG [RS:1;adda16083778:45169 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/93be71cd-4dcf-2478-daf8-a2306a879141/e0862929-e242-795d-1e0b-fe7766981a5b/oldWALs 2024-11-23T21:54:47,197 INFO [RS:1;adda16083778:45169 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL adda16083778%2C45169%2C1732398883012:(num 1732398883701) 2024-11-23T21:54:47,197 DEBUG [RS:1;adda16083778:45169 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T21:54:47,197 INFO [RS:1;adda16083778:45169 {}] regionserver.LeaseManager(133): Closed leases 2024-11-23T21:54:47,197 INFO [RS:1;adda16083778:45169 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-23T21:54:47,198 INFO [RS:1;adda16083778:45169 {}] hbase.ChoreService(370): Chore service for: regionserver/adda16083778:0 had [ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS] on shutdown 2024-11-23T21:54:47,198 INFO [RS:1;adda16083778:45169 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-23T21:54:47,198 INFO [regionserver/adda16083778:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-23T21:54:47,198 INFO [RS:1;adda16083778:45169 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:45169 2024-11-23T21:54:47,224 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45169-0x1016afab2c80002, quorum=127.0.0.1:51781, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/adda16083778,45169,1732398883012 2024-11-23T21:54:47,224 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33953-0x1016afab2c80000, quorum=127.0.0.1:51781, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-23T21:54:47,224 INFO [RS:1;adda16083778:45169 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-23T21:54:47,225 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [adda16083778,45169,1732398883012] 2024-11-23T21:54:47,241 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/adda16083778,45169,1732398883012 already deleted, retry=false 2024-11-23T21:54:47,241 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; adda16083778,45169,1732398883012 expired; onlineServers=0 2024-11-23T21:54:47,242 INFO [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master 'adda16083778,33953,1732398882834' ***** 2024-11-23T21:54:47,242 INFO [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0 2024-11-23T21:54:47,242 INFO [M:0;adda16083778:33953 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-23T21:54:47,242 INFO [M:0;adda16083778:33953 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-23T21:54:47,242 DEBUG [M:0;adda16083778:33953 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2024-11-23T21:54:47,242 DEBUG [M:0;adda16083778:33953 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2024-11-23T21:54:47,242 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2024-11-23T21:54:47,242 DEBUG [master/adda16083778:0:becomeActiveMaster-HFileCleaner.large.0-1732398883377 {}] cleaner.HFileCleaner(306): Exit Thread[master/adda16083778:0:becomeActiveMaster-HFileCleaner.large.0-1732398883377,5,FailOnTimeoutGroup] 2024-11-23T21:54:47,243 DEBUG [master/adda16083778:0:becomeActiveMaster-HFileCleaner.small.0-1732398883378 {}] cleaner.HFileCleaner(306): Exit Thread[master/adda16083778:0:becomeActiveMaster-HFileCleaner.small.0-1732398883378,5,FailOnTimeoutGroup] 2024-11-23T21:54:47,243 INFO [M:0;adda16083778:33953 {}] hbase.ChoreService(370): Chore service for: master/adda16083778:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown 2024-11-23T21:54:47,243 INFO [M:0;adda16083778:33953 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-23T21:54:47,244 DEBUG [M:0;adda16083778:33953 {}] master.HMaster(1795): Stopping service threads 2024-11-23T21:54:47,244 INFO [M:0;adda16083778:33953 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2024-11-23T21:54:47,244 INFO [M:0;adda16083778:33953 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-23T21:54:47,244 INFO [M:0;adda16083778:33953 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-23T21:54:47,245 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2024-11-23T21:54:47,250 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33953-0x1016afab2c80000, quorum=127.0.0.1:51781, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-11-23T21:54:47,250 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33953-0x1016afab2c80000, quorum=127.0.0.1:51781, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-23T21:54:47,250 DEBUG [M:0;adda16083778:33953 {}] zookeeper.ZKUtil(347): master:33953-0x1016afab2c80000, quorum=127.0.0.1:51781, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error) 2024-11-23T21:54:47,250 WARN [M:0;adda16083778:33953 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null 2024-11-23T21:54:47,250 INFO [M:0;adda16083778:33953 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/93be71cd-4dcf-2478-daf8-a2306a879141/e0862929-e242-795d-1e0b-fe7766981a5b/.lastflushedseqids 2024-11-23T21:54:47,260 INFO [M:0;adda16083778:33953 {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-23T21:54:47,260 INFO [M:0;adda16083778:33953 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2024-11-23T21:54:47,260 DEBUG [M:0;adda16083778:33953 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-23T21:54:47,260 INFO [M:0;adda16083778:33953 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-23T21:54:47,260 DEBUG [M:0;adda16083778:33953 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-23T21:54:47,260 DEBUG [M:0;adda16083778:33953 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-23T21:54:47,260 DEBUG [M:0;adda16083778:33953 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-23T21:54:47,261 INFO [M:0;adda16083778:33953 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=34.95 KB heapSize=44.17 KB 2024-11-23T21:54:47,271 DEBUG [M:0;adda16083778:33953 {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 8c323e2fb9eb420b8a2f7f361219362a 2024-11-23T21:54:47,288 DEBUG [M:0;adda16083778:33953 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/93be71cd-4dcf-2478-daf8-a2306a879141/e0862929-e242-795d-1e0b-fe7766981a5b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/8c323e2fb9eb420b8a2f7f361219362a is 82, key is hbase:meta,,1/info:regioninfo/1732398886643/Put/seqid=0 2024-11-23T21:54:47,288 INFO [M:0;adda16083778:33953 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1008 B at sequenceid=90 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/93be71cd-4dcf-2478-daf8-a2306a879141/e0862929-e242-795d-1e0b-fe7766981a5b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/8c323e2fb9eb420b8a2f7f361219362a 2024-11-23T21:54:47,295 DEBUG [M:0;adda16083778:33953 {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 83221c3756744e758fc3d27499c31511 2024-11-23T21:54:47,307 DEBUG [M:0;adda16083778:33953 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/93be71cd-4dcf-2478-daf8-a2306a879141/e0862929-e242-795d-1e0b-fe7766981a5b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/83221c3756744e758fc3d27499c31511 is 783, key is \x00\x00\x00\x00\x00\x00\x00\x04/proc:d/1732398884536/Put/seqid=0 2024-11-23T21:54:47,308 INFO [M:0;adda16083778:33953 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.73 KB at sequenceid=90 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/93be71cd-4dcf-2478-daf8-a2306a879141/e0862929-e242-795d-1e0b-fe7766981a5b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/83221c3756744e758fc3d27499c31511 2024-11-23T21:54:47,315 DEBUG [M:0;adda16083778:33953 {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 3b40e3a83ef648849570d5c8d853f315 2024-11-23T21:54:47,327 DEBUG [M:0;adda16083778:33953 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/93be71cd-4dcf-2478-daf8-a2306a879141/e0862929-e242-795d-1e0b-fe7766981a5b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/3b40e3a83ef648849570d5c8d853f315 is 69, key is adda16083778,34911,1732398886341/rs:state/1732398886408/Put/seqid=0 2024-11-23T21:54:47,328 INFO [M:0;adda16083778:33953 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=249 B at sequenceid=90 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/93be71cd-4dcf-2478-daf8-a2306a879141/e0862929-e242-795d-1e0b-fe7766981a5b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/3b40e3a83ef648849570d5c8d853f315 2024-11-23T21:54:47,329 INFO [M:0;adda16083778:33953 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 3b40e3a83ef648849570d5c8d853f315 2024-11-23T21:54:47,330 DEBUG [M:0;adda16083778:33953 {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/93be71cd-4dcf-2478-daf8-a2306a879141/e0862929-e242-795d-1e0b-fe7766981a5b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/8c323e2fb9eb420b8a2f7f361219362a as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/93be71cd-4dcf-2478-daf8-a2306a879141/e0862929-e242-795d-1e0b-fe7766981a5b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/8c323e2fb9eb420b8a2f7f361219362a 2024-11-23T21:54:47,331 INFO [M:0;adda16083778:33953 {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/93be71cd-4dcf-2478-daf8-a2306a879141/e0862929-e242-795d-1e0b-fe7766981a5b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/8c323e2fb9eb420b8a2f7f361219362a, entries=14, sequenceid=90, filesize=5.9 K 2024-11-23T21:54:47,332 DEBUG [M:0;adda16083778:33953 {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/93be71cd-4dcf-2478-daf8-a2306a879141/e0862929-e242-795d-1e0b-fe7766981a5b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/83221c3756744e758fc3d27499c31511 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/93be71cd-4dcf-2478-daf8-a2306a879141/e0862929-e242-795d-1e0b-fe7766981a5b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/83221c3756744e758fc3d27499c31511 2024-11-23T21:54:47,333 INFO [M:0;adda16083778:33953 {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/93be71cd-4dcf-2478-daf8-a2306a879141/e0862929-e242-795d-1e0b-fe7766981a5b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/83221c3756744e758fc3d27499c31511, entries=11, sequenceid=90, filesize=7.8 K 2024-11-23T21:54:47,333 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45169-0x1016afab2c80002, quorum=127.0.0.1:51781, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-23T21:54:47,333 INFO [RS:1;adda16083778:45169 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-23T21:54:47,333 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45169-0x1016afab2c80002, quorum=127.0.0.1:51781, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-23T21:54:47,333 INFO [RS:1;adda16083778:45169 {}] regionserver.HRegionServer(1031): Exiting; stopping=adda16083778,45169,1732398883012; zookeeper connection closed. 2024-11-23T21:54:47,333 DEBUG [M:0;adda16083778:33953 {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/93be71cd-4dcf-2478-daf8-a2306a879141/e0862929-e242-795d-1e0b-fe7766981a5b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/3b40e3a83ef648849570d5c8d853f315 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/93be71cd-4dcf-2478-daf8-a2306a879141/e0862929-e242-795d-1e0b-fe7766981a5b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/3b40e3a83ef648849570d5c8d853f315 2024-11-23T21:54:47,333 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@669cc7b7 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@669cc7b7 2024-11-23T21:54:47,334 INFO [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 3 regionserver(s) complete 2024-11-23T21:54:47,335 INFO [M:0;adda16083778:33953 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 3b40e3a83ef648849570d5c8d853f315 2024-11-23T21:54:47,335 INFO [M:0;adda16083778:33953 {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/93be71cd-4dcf-2478-daf8-a2306a879141/e0862929-e242-795d-1e0b-fe7766981a5b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/3b40e3a83ef648849570d5c8d853f315, entries=3, sequenceid=90, filesize=5.3 K 2024-11-23T21:54:47,336 INFO [M:0;adda16083778:33953 {}] regionserver.HRegion(3140): Finished flush of dataSize ~34.95 KB/35792, heapSize ~43.88 KB/44928, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 76ms, sequenceid=90, compaction requested=false 2024-11-23T21:54:47,338 INFO [M:0;adda16083778:33953 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-23T21:54:47,338 DEBUG [M:0;adda16083778:33953 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1732398887260Disabling compacts and flushes for region at 1732398887260Disabling writes for close at 1732398887260Obtaining lock to block concurrent updates at 1732398887261 (+1 ms)Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1732398887261Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=35792, getHeapSize=45168, getOffHeapSize=0, getCellsCount=111 at 1732398887261Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1732398887262 (+1 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1732398887262Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1732398887287 (+25 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1732398887287Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1732398887290 (+3 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1732398887307 (+17 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1732398887307Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1732398887310 (+3 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1732398887326 (+16 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1732398887326Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@34ebf3d4: reopening flushed file at 1732398887329 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@1d9bc5b6: reopening flushed file at 1732398887331 (+2 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@e627ff7: reopening flushed file at 1732398887333 (+2 ms)Finished flush of dataSize ~34.95 KB/35792, heapSize ~43.88 KB/44928, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 76ms, sequenceid=90, compaction requested=false at 1732398887336 (+3 ms)Writing region close event to WAL at 1732398887337 (+1 ms)Closed at 1732398887338 (+1 ms) 2024-11-23T21:54:47,339 INFO [M:0;adda16083778:33953 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down. 2024-11-23T21:54:47,339 INFO [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-23T21:54:47,339 INFO [M:0;adda16083778:33953 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:33953 2024-11-23T21:54:47,339 INFO [M:0;adda16083778:33953 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-23T21:54:47,482 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33953-0x1016afab2c80000, quorum=127.0.0.1:51781, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-23T21:54:47,482 INFO [M:0;adda16083778:33953 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-23T21:54:47,483 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33953-0x1016afab2c80000, quorum=127.0.0.1:51781, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-23T21:54:47,486 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers 2024-11-23T21:54:47,488 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down 2024-11-23T21:54:47,488 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-23T21:54:47,493 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: io.hfile.TestBlockEvictionOnRegionMovement#testBlockEvictionOnGracefulStop Thread=70 (was 50) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-8 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HMaster-EventLoopGroup-6-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-7-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-8-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-7-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-8-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HMaster-EventLoopGroup-6-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-8-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-6 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-9-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-9-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-7-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-7 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-5 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HMaster-EventLoopGroup-6-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=382 (was 346) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=384 (was 418), ProcessCount=11 (was 11), AvailableMemoryMB=8402 (was 8509)