2024-11-21 19:48:52,316 main DEBUG Apache Log4j Core 2.17.2 initializing configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@15a34df2 2024-11-21 19:48:52,333 main DEBUG Took 0.015415 seconds to load 1 plugins from package org.apache.hadoop.hbase.logging 2024-11-21 19:48:52,333 main DEBUG PluginManager 'Core' found 129 plugins 2024-11-21 19:48:52,333 main DEBUG PluginManager 'Level' found 0 plugins 2024-11-21 19:48:52,334 main DEBUG PluginManager 'Lookup' found 16 plugins 2024-11-21 19:48:52,335 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-21 19:48:52,342 main DEBUG PluginManager 'TypeConverter' found 26 plugins 2024-11-21 19:48:52,353 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.util.MBeans", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-21 19:48:52,354 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-21 19:48:52,355 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.logging.TestJul2Slf4j", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-21 19:48:52,355 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-21 19:48:52,356 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.zookeeper", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-21 19:48:52,356 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-21 19:48:52,357 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSinkAdapter", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-21 19:48:52,357 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-21 19:48:52,357 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSystemImpl", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-21 19:48:52,358 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-21 19:48:52,358 main DEBUG LoggerConfig$Builder(additivity="false", level="WARN", levelAndRefs="null", name="org.apache.directory", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-21 19:48:52,359 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-21 19:48:52,359 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.ipc.FailedServers", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-21 19:48:52,360 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-21 19:48:52,360 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsConfig", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-21 19:48:52,360 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-21 19:48:52,361 main DEBUG LoggerConfig$Builder(additivity="null", level="INFO", levelAndRefs="null", name="org.apache.hadoop.hbase.ScheduledChore", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-21 19:48:52,361 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-21 19:48:52,361 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.regionserver.RSRpcServices", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-21 19:48:52,362 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-21 19:48:52,362 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-21 19:48:52,362 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-21 19:48:52,363 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-21 19:48:52,363 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-21 19:48:52,363 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hbase.thirdparty.io.netty.channel", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-21 19:48:52,364 main DEBUG Building Plugin[name=root, class=org.apache.logging.log4j.core.config.LoggerConfig$RootLogger]. 2024-11-21 19:48:52,365 main DEBUG LoggerConfig$RootLogger$Builder(additivity="null", level="null", levelAndRefs="INFO,Console", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-21 19:48:52,366 main DEBUG Building Plugin[name=loggers, class=org.apache.logging.log4j.core.config.LoggersPlugin]. 2024-11-21 19:48:52,367 main DEBUG createLoggers(={org.apache.hadoop.metrics2.util.MBeans, org.apache.hadoop.hbase.logging.TestJul2Slf4j, org.apache.zookeeper, org.apache.hadoop.metrics2.impl.MetricsSinkAdapter, org.apache.hadoop.metrics2.impl.MetricsSystemImpl, org.apache.directory, org.apache.hadoop.hbase.ipc.FailedServers, org.apache.hadoop.metrics2.impl.MetricsConfig, org.apache.hadoop.hbase.ScheduledChore, org.apache.hadoop.hbase.regionserver.RSRpcServices, org.apache.hadoop, org.apache.hadoop.hbase, org.apache.hbase.thirdparty.io.netty.channel, root}) 2024-11-21 19:48:52,368 main DEBUG Building Plugin[name=layout, class=org.apache.logging.log4j.core.layout.PatternLayout]. 2024-11-21 19:48:52,369 main DEBUG PatternLayout$Builder(pattern="%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n", PatternSelector=null, Configuration(PropertiesConfig), Replace=null, charset="null", alwaysWriteExceptions="null", disableAnsi="null", noConsoleNoAnsi="null", header="null", footer="null") 2024-11-21 19:48:52,369 main DEBUG PluginManager 'Converter' found 47 plugins 2024-11-21 19:48:52,377 main DEBUG Building Plugin[name=appender, class=org.apache.hadoop.hbase.logging.HBaseTestAppender]. 2024-11-21 19:48:52,380 main DEBUG HBaseTestAppender$Builder(target="SYSTEM_ERR", maxSize="1G", bufferedIo="null", bufferSize="null", immediateFlush="null", ignoreExceptions="null", PatternLayout(%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n), name="Console", Configuration(PropertiesConfig), Filter=null, ={}) 2024-11-21 19:48:52,381 main DEBUG Starting HBaseTestOutputStreamManager SYSTEM_ERR 2024-11-21 19:48:52,381 main DEBUG Building Plugin[name=appenders, class=org.apache.logging.log4j.core.config.AppendersPlugin]. 2024-11-21 19:48:52,382 main DEBUG createAppenders(={Console}) 2024-11-21 19:48:52,382 main DEBUG Configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@15a34df2 initialized 2024-11-21 19:48:52,383 main DEBUG Starting configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@15a34df2 2024-11-21 19:48:52,383 main DEBUG Started configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@15a34df2 OK. 2024-11-21 19:48:52,383 main DEBUG Shutting down OutputStreamManager SYSTEM_OUT.false.false-1 2024-11-21 19:48:52,384 main DEBUG OutputStream closed 2024-11-21 19:48:52,384 main DEBUG Shut down OutputStreamManager SYSTEM_OUT.false.false-1, all resources released: true 2024-11-21 19:48:52,384 main DEBUG Appender DefaultConsole-1 stopped with status true 2024-11-21 19:48:52,384 main DEBUG Stopped org.apache.logging.log4j.core.config.DefaultConfiguration@4f2b503c OK 2024-11-21 19:48:52,448 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6 2024-11-21 19:48:52,450 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=StatusLogger 2024-11-21 19:48:52,451 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=ContextSelector 2024-11-21 19:48:52,452 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name= 2024-11-21 19:48:52,452 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.directory 2024-11-21 19:48:52,453 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSinkAdapter 2024-11-21 19:48:52,453 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.zookeeper 2024-11-21 19:48:52,453 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.logging.TestJul2Slf4j 2024-11-21 19:48:52,454 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSystemImpl 2024-11-21 19:48:52,454 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.util.MBeans 2024-11-21 19:48:52,454 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase 2024-11-21 19:48:52,454 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop 2024-11-21 19:48:52,455 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ipc.FailedServers 2024-11-21 19:48:52,455 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.regionserver.RSRpcServices 2024-11-21 19:48:52,455 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsConfig 2024-11-21 19:48:52,456 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hbase.thirdparty.io.netty.channel 2024-11-21 19:48:52,456 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ScheduledChore 2024-11-21 19:48:52,457 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Appenders,name=Console 2024-11-21 19:48:52,459 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-21 19:48:52,459 main DEBUG Reconfiguration complete for context[name=1dbd16a6] at URI jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-logging/target/hbase-logging-4.0.0-alpha-1-SNAPSHOT-tests.jar!/log4j2.properties (org.apache.logging.log4j.core.LoggerContext@31c7528f) with optional ClassLoader: null 2024-11-21 19:48:52,459 main DEBUG Shutdown hook enabled. Registering a new one. 2024-11-21 19:48:52,460 main DEBUG LoggerContext[name=1dbd16a6, org.apache.logging.log4j.core.LoggerContext@31c7528f] started OK. 2024-11-21T19:48:52,473 INFO [main {}] hbase.HBaseClassTestRule(94): Test class org.apache.hadoop.hbase.io.hfile.TestBlockEvictionOnRegionMovement timeout: 13 mins 2024-11-21 19:48:52,476 main DEBUG AsyncLogger.ThreadNameStrategy=UNCACHED (user specified null, default is UNCACHED) 2024-11-21 19:48:52,476 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-21T19:48:52,684 DEBUG [main {}] hbase.HBaseTestingUtil(323): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0b774921-3780-c936-4fe1-1d1dcb4ce861 2024-11-21T19:48:52,705 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: io.hfile.TestBlockEvictionOnRegionMovement#testBlockEvictionOnRegionMove Thread=12, OpenFileDescriptor=287, MaxFileDescriptor=1048576, SystemLoadAverage=504, ProcessCount=11, AvailableMemoryMB=9746 2024-11-21T19:48:52,801 WARN [Time-limited test {}] util.NativeCodeLoader(60): Unable to load native-hadoop library for your platform... using builtin-java classes where applicable 2024-11-21T19:48:52,889 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-21T19:48:52,905 INFO [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0b774921-3780-c936-4fe1-1d1dcb4ce861/cluster_26fb76e4-9a4a-7965-cb45-14bd9646fdd1, deleteOnExit=true 2024-11-21T19:48:52,999 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0b774921-3780-c936-4fe1-1d1dcb4ce861/cluster_26fb76e4-9a4a-7965-cb45-14bd9646fdd1/zookeeper_0, clientPort=50077, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0b774921-3780-c936-4fe1-1d1dcb4ce861/cluster_26fb76e4-9a4a-7965-cb45-14bd9646fdd1/zookeeper_0/version-2, dataDirSize=457 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0b774921-3780-c936-4fe1-1d1dcb4ce861/cluster_26fb76e4-9a4a-7965-cb45-14bd9646fdd1/zookeeper_0/version-2, dataLogSize=457 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0 2024-11-21T19:48:53,011 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=50077 2024-11-21T19:48:53,011 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-21T19:48:53,099 INFO [Time-limited test {}] util.FSUtils(489): Created version file at /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0b774921-3780-c936-4fe1-1d1dcb4ce861/545083c6-d774-d2ca-7409-0fbc2c22cf1a with version=8 2024-11-21T19:48:53,100 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1139): Setting hbase.fs.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0b774921-3780-c936-4fe1-1d1dcb4ce861/545083c6-d774-d2ca-7409-0fbc2c22cf1a/hbase-staging 2024-11-21T19:48:53,195 DEBUG [Time-limited test {}] channel.MultithreadEventLoopGroup(44): -Dio.netty.eventLoopThreads: 16 2024-11-21T19:48:53,469 INFO [Time-limited test {}] client.ConnectionUtils(128): master/7b96b36745c6:0 server-side Connection retries=45 2024-11-21T19:48:53,479 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-21T19:48:53,480 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-21T19:48:53,487 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-21T19:48:53,488 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-21T19:48:53,488 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-21T19:48:53,622 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService 2024-11-21T19:48:53,706 WARN [Time-limited test {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-21T19:48:53,742 INFO [Time-limited test {}] metrics.MetricRegistriesLoader(60): Loaded MetricRegistries class org.apache.hadoop.hbase.metrics.impl.MetricRegistriesImpl 2024-11-21T19:48:53,752 DEBUG [Time-limited test {}] util.ClassSize(228): Using Unsafe to estimate memory layout 2024-11-21T19:48:53,755 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-21T19:48:53,796 DEBUG [Time-limited test {}] channel.DefaultChannelId(84): -Dio.netty.processId: 21471 (auto-detected) 2024-11-21T19:48:53,797 DEBUG [Time-limited test {}] channel.DefaultChannelId(106): -Dio.netty.machineId: 02:42:ac:ff:fe:11:00:02 (auto-detected) 2024-11-21T19:48:53,828 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:39849 2024-11-21T19:48:53,853 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:39849 connecting to ZooKeeper ensemble=127.0.0.1:50077 2024-11-21T19:48:53,886 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:398490x0, quorum=127.0.0.1:50077, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-21T19:48:53,889 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:39849-0x10116d829200000 connected 2024-11-21T19:48:53,915 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-21T19:48:53,916 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-21T19:48:53,923 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:39849-0x10116d829200000, quorum=127.0.0.1:50077, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-21T19:48:53,927 INFO [Time-limited test {}] master.HMaster(525): hbase.rootdir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0b774921-3780-c936-4fe1-1d1dcb4ce861/545083c6-d774-d2ca-7409-0fbc2c22cf1a, hbase.cluster.distributed=false 2024-11-21T19:48:53,949 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:39849-0x10116d829200000, quorum=127.0.0.1:50077, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-21T19:48:53,954 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=39849 2024-11-21T19:48:53,955 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=39849 2024-11-21T19:48:53,956 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=39849 2024-11-21T19:48:53,958 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=39849 2024-11-21T19:48:53,959 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=39849 2024-11-21T19:48:54,042 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/7b96b36745c6:0 server-side Connection retries=45 2024-11-21T19:48:54,044 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-21T19:48:54,044 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-21T19:48:54,044 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-21T19:48:54,045 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-21T19:48:54,045 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-21T19:48:54,048 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-21T19:48:54,050 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-21T19:48:54,051 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:33427 2024-11-21T19:48:54,053 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:33427 connecting to ZooKeeper ensemble=127.0.0.1:50077 2024-11-21T19:48:54,053 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-21T19:48:54,054 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-21T19:48:54,058 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:334270x0, quorum=127.0.0.1:50077, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-21T19:48:54,058 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:33427-0x10116d829200001 connected 2024-11-21T19:48:54,059 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:33427-0x10116d829200001, quorum=127.0.0.1:50077, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-21T19:48:54,064 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-21T19:48:54,075 INFO [Time-limited test {}] bucket.FileIOEngine(99): Allocating cache 400 MB, on the path:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0b774921-3780-c936-4fe1-1d1dcb4ce861/bucket.cache 2024-11-21T19:48:54,081 INFO [Time-limited test {}] bucket.BucketCache(364): Instantiating BucketCache with acceptableFactor: 0.95, minFactor: 0.85, extraFreeFactor: 0.1, singleFactor: 0.25, multiFactor: 0.5, memoryFactor: 0.25, useStrongRef: false 2024-11-21T19:48:54,082 INFO [Time-limited test {}] bucket.BucketCache(478): Starting BucketCachePersisterThread 2024-11-21T19:48:54,083 INFO [Time-limited test {}] bucket.BucketCachePersister(38): BucketCachePersister started with interval: 100 2024-11-21T19:48:54,085 INFO [Thread-10 {}] bucket.BucketCache(1530): Started retrieving bucket cache from file 2024-11-21T19:48:54,085 WARN [Thread-10 {}] bucket.BucketCache(1533): Persistence file missing! It's ok if it's first run after enabling persistent cache. 2024-11-21T19:48:54,090 INFO [Time-limited test {}] bucket.BucketCache(403): Started bucket cache; ioengine=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0b774921-3780-c936-4fe1-1d1dcb4ce861/bucket.cache, capacity=400 MB, blockSize=64 KB, writerThreadNum=3, writerQLen=64, persistencePath=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0b774921-3780-c936-4fe1-1d1dcb4ce861/bucket.persistence, bucketAllocator=org.apache.hadoop.hbase.io.hfile.bucket.BucketAllocator 2024-11-21T19:48:54,095 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-21T19:48:54,096 INFO [Thread-10 {}] bucket.BucketAllocator(343): Cache totalSize=418148352, buckets=199, bucket capacity=2101248=(4*525312)=(FEWEST_ITEMS_IN_BUCKET*(largest configured bucketcache size)) 2024-11-21T19:48:54,096 INFO [Thread-10 {}] bucket.BucketCache(413): Persistent bucket cache recovery from /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0b774921-3780-c936-4fe1-1d1dcb4ce861/bucket.persistence is complete. 2024-11-21T19:48:54,097 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:33427-0x10116d829200001, quorum=127.0.0.1:50077, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-21T19:48:54,102 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:33427-0x10116d829200001, quorum=127.0.0.1:50077, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-21T19:48:54,105 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=33427 2024-11-21T19:48:54,107 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=33427 2024-11-21T19:48:54,108 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=33427 2024-11-21T19:48:54,108 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=33427 2024-11-21T19:48:54,109 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=33427 2024-11-21T19:48:54,127 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/7b96b36745c6:0 server-side Connection retries=45 2024-11-21T19:48:54,127 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-21T19:48:54,127 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-21T19:48:54,127 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-21T19:48:54,128 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-21T19:48:54,128 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-21T19:48:54,128 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-21T19:48:54,128 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-21T19:48:54,129 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:43053 2024-11-21T19:48:54,131 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:43053 connecting to ZooKeeper ensemble=127.0.0.1:50077 2024-11-21T19:48:54,131 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-21T19:48:54,132 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-21T19:48:54,135 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:430530x0, quorum=127.0.0.1:50077, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-21T19:48:54,136 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:43053-0x10116d829200002 connected 2024-11-21T19:48:54,136 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:43053-0x10116d829200002, quorum=127.0.0.1:50077, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-21T19:48:54,137 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-21T19:48:54,139 INFO [Time-limited test {}] bucket.FileIOEngine(99): Allocating cache 400 MB, on the path:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0b774921-3780-c936-4fe1-1d1dcb4ce861/bucket.cache 2024-11-21T19:48:54,140 INFO [Time-limited test {}] bucket.BucketCache(364): Instantiating BucketCache with acceptableFactor: 0.95, minFactor: 0.85, extraFreeFactor: 0.1, singleFactor: 0.25, multiFactor: 0.5, memoryFactor: 0.25, useStrongRef: false 2024-11-21T19:48:54,140 INFO [Time-limited test {}] bucket.BucketCache(478): Starting BucketCachePersisterThread 2024-11-21T19:48:54,140 INFO [Time-limited test {}] bucket.BucketCachePersister(38): BucketCachePersister started with interval: 100 2024-11-21T19:48:54,141 INFO [Thread-11 {}] bucket.BucketCache(1530): Started retrieving bucket cache from file 2024-11-21T19:48:54,141 INFO [Time-limited test {}] bucket.BucketCache(403): Started bucket cache; ioengine=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0b774921-3780-c936-4fe1-1d1dcb4ce861/bucket.cache, capacity=400 MB, blockSize=64 KB, writerThreadNum=3, writerQLen=64, persistencePath=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0b774921-3780-c936-4fe1-1d1dcb4ce861/bucket.persistence, bucketAllocator=org.apache.hadoop.hbase.io.hfile.bucket.BucketAllocator 2024-11-21T19:48:54,141 WARN [Thread-11 {}] bucket.BucketCache(1533): Persistence file missing! It's ok if it's first run after enabling persistent cache. 2024-11-21T19:48:54,141 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-21T19:48:54,142 INFO [Thread-11 {}] bucket.BucketAllocator(343): Cache totalSize=418148352, buckets=199, bucket capacity=2101248=(4*525312)=(FEWEST_ITEMS_IN_BUCKET*(largest configured bucketcache size)) 2024-11-21T19:48:54,142 INFO [Thread-11 {}] bucket.BucketCache(413): Persistent bucket cache recovery from /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0b774921-3780-c936-4fe1-1d1dcb4ce861/bucket.persistence is complete. 2024-11-21T19:48:54,142 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:43053-0x10116d829200002, quorum=127.0.0.1:50077, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-21T19:48:54,145 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:43053-0x10116d829200002, quorum=127.0.0.1:50077, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-21T19:48:54,149 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=43053 2024-11-21T19:48:54,149 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=43053 2024-11-21T19:48:54,155 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=43053 2024-11-21T19:48:54,156 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=43053 2024-11-21T19:48:54,157 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=43053 2024-11-21T19:48:54,180 DEBUG [M:0;7b96b36745c6:39849 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;7b96b36745c6:39849 2024-11-21T19:48:54,182 INFO [master/7b96b36745c6:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/7b96b36745c6,39849,1732218533264 2024-11-21T19:48:54,189 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33427-0x10116d829200001, quorum=127.0.0.1:50077, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-21T19:48:54,189 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39849-0x10116d829200000, quorum=127.0.0.1:50077, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-21T19:48:54,189 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43053-0x10116d829200002, quorum=127.0.0.1:50077, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-21T19:48:54,191 DEBUG [master/7b96b36745c6:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:39849-0x10116d829200000, quorum=127.0.0.1:50077, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/7b96b36745c6,39849,1732218533264 2024-11-21T19:48:54,214 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43053-0x10116d829200002, quorum=127.0.0.1:50077, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-21T19:48:54,214 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33427-0x10116d829200001, quorum=127.0.0.1:50077, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-21T19:48:54,214 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39849-0x10116d829200000, quorum=127.0.0.1:50077, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-21T19:48:54,214 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43053-0x10116d829200002, quorum=127.0.0.1:50077, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-21T19:48:54,214 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33427-0x10116d829200001, quorum=127.0.0.1:50077, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-21T19:48:54,216 DEBUG [master/7b96b36745c6:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:39849-0x10116d829200000, quorum=127.0.0.1:50077, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-11-21T19:48:54,217 INFO [master/7b96b36745c6:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/7b96b36745c6,39849,1732218533264 from backup master directory 2024-11-21T19:48:54,221 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39849-0x10116d829200000, quorum=127.0.0.1:50077, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/7b96b36745c6,39849,1732218533264 2024-11-21T19:48:54,221 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43053-0x10116d829200002, quorum=127.0.0.1:50077, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-21T19:48:54,221 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39849-0x10116d829200000, quorum=127.0.0.1:50077, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-21T19:48:54,221 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33427-0x10116d829200001, quorum=127.0.0.1:50077, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-21T19:48:54,222 WARN [master/7b96b36745c6:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-21T19:48:54,222 INFO [master/7b96b36745c6:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=7b96b36745c6,39849,1732218533264 2024-11-21T19:48:54,225 INFO [master/7b96b36745c6:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating data MemStoreChunkPool with chunk size 2 MB, max count 396, initial count 0 2024-11-21T19:48:54,227 INFO [master/7b96b36745c6:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating index MemStoreChunkPool with chunk size 204.80 KB, max count 440, initial count 0 2024-11-21T19:48:54,237 DEBUG [master/7b96b36745c6:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0b774921-3780-c936-4fe1-1d1dcb4ce861/545083c6-d774-d2ca-7409-0fbc2c22cf1a/hbase.id] with ID: 5d6261fd-cc48-4e27-9635-c975d4b37434 2024-11-21T19:48:54,237 DEBUG [master/7b96b36745c6:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0b774921-3780-c936-4fe1-1d1dcb4ce861/545083c6-d774-d2ca-7409-0fbc2c22cf1a/.tmp/hbase.id 2024-11-21T19:48:54,244 DEBUG [master/7b96b36745c6:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0b774921-3780-c936-4fe1-1d1dcb4ce861/545083c6-d774-d2ca-7409-0fbc2c22cf1a/.tmp/hbase.id]:[file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0b774921-3780-c936-4fe1-1d1dcb4ce861/545083c6-d774-d2ca-7409-0fbc2c22cf1a/hbase.id] 2024-11-21T19:48:54,284 DEBUG [master/7b96b36745c6:0:becomeActiveMaster {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-21T19:48:54,289 INFO [master/7b96b36745c6:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem. 2024-11-21T19:48:54,310 INFO [master/7b96b36745c6:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 19ms. 2024-11-21T19:48:54,315 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39849-0x10116d829200000, quorum=127.0.0.1:50077, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-21T19:48:54,315 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33427-0x10116d829200001, quorum=127.0.0.1:50077, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-21T19:48:54,315 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43053-0x10116d829200002, quorum=127.0.0.1:50077, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-21T19:48:54,340 INFO [master/7b96b36745c6:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-21T19:48:54,343 INFO [master/7b96b36745c6:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-11-21T19:48:54,391 DEBUG [master/7b96b36745c6:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(244): No decryptEncryptedDataEncryptionKey method in DFSClient, should be hadoop version with HDFS-12396 java.lang.NoSuchMethodException: org.apache.hadoop.hdfs.DFSClient.decryptEncryptedDataEncryptionKey(org.apache.hadoop.fs.FileEncryptionInfo) at java.lang.Class.getDeclaredMethod(Class.java:2675) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelperWithoutHDFS12396(FanOutOneBlockAsyncDFSOutputSaslHelper.java:183) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelper(FanOutOneBlockAsyncDFSOutputSaslHelper.java:242) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.(FanOutOneBlockAsyncDFSOutputSaslHelper.java:253) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.lang.Class.forName0(Native Method) ~[?:?] at java.lang.Class.forName(Class.java:375) ~[?:?] at org.apache.hadoop.hbase.wal.AsyncFSWALProvider.load(AsyncFSWALProvider.java:150) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.getProviderClass(WALFactory.java:174) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:262) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:231) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegion.create(MasterRegion.java:400) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegionFactory.create(MasterRegionFactory.java:135) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.finishActiveMasterInitialization(HMaster.java:1003) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.startActiveMasterManager(HMaster.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.lambda$run$0(HMaster.java:613) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.lambda$tracedRunnable$2(TraceUtil.java:155) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T19:48:54,397 INFO [master/7b96b36745c6:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-21T19:48:54,457 INFO [master/7b96b36745c6:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0b774921-3780-c936-4fe1-1d1dcb4ce861/545083c6-d774-d2ca-7409-0fbc2c22cf1a/MasterData/data/master/store 2024-11-21T19:48:54,484 INFO [master/7b96b36745c6:0:becomeActiveMaster {}] throttle.StoreHotnessProtector(112): StoreHotnessProtector is disabled. Set hbase.region.store.parallel.put.limit > 0 to enable, which may help mitigate load under heavy write pressure. 2024-11-21T19:48:54,488 DEBUG [master/7b96b36745c6:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-21T19:48:54,490 DEBUG [master/7b96b36745c6:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-21T19:48:54,490 INFO [master/7b96b36745c6:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-21T19:48:54,491 DEBUG [master/7b96b36745c6:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-21T19:48:54,493 DEBUG [master/7b96b36745c6:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-21T19:48:54,493 DEBUG [master/7b96b36745c6:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-21T19:48:54,493 INFO [master/7b96b36745c6:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-21T19:48:54,494 DEBUG [master/7b96b36745c6:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1732218534490Disabling compacts and flushes for region at 1732218534490Disabling writes for close at 1732218534493 (+3 ms)Writing region close event to WAL at 1732218534493Closed at 1732218534493 2024-11-21T19:48:54,494 WARN [master/7b96b36745c6:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0b774921-3780-c936-4fe1-1d1dcb4ce861/545083c6-d774-d2ca-7409-0fbc2c22cf1a/MasterData/data/master/store/.initializing 2024-11-21T19:48:54,494 DEBUG [master/7b96b36745c6:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0b774921-3780-c936-4fe1-1d1dcb4ce861/545083c6-d774-d2ca-7409-0fbc2c22cf1a/MasterData/WALs/7b96b36745c6,39849,1732218533264 2024-11-21T19:48:54,507 INFO [master/7b96b36745c6:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-21T19:48:54,525 INFO [master/7b96b36745c6:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=7b96b36745c6%2C39849%2C1732218533264, suffix=, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0b774921-3780-c936-4fe1-1d1dcb4ce861/545083c6-d774-d2ca-7409-0fbc2c22cf1a/MasterData/WALs/7b96b36745c6,39849,1732218533264, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0b774921-3780-c936-4fe1-1d1dcb4ce861/545083c6-d774-d2ca-7409-0fbc2c22cf1a/MasterData/oldWALs, maxLogs=10 2024-11-21T19:48:54,568 INFO [master/7b96b36745c6:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0b774921-3780-c936-4fe1-1d1dcb4ce861/545083c6-d774-d2ca-7409-0fbc2c22cf1a/MasterData/WALs/7b96b36745c6,39849,1732218533264/7b96b36745c6%2C39849%2C1732218533264.1732218534531 2024-11-21T19:48:54,569 DEBUG [master/7b96b36745c6:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-11-21T19:48:54,570 DEBUG [master/7b96b36745c6:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-11-21T19:48:54,571 DEBUG [master/7b96b36745c6:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-21T19:48:54,575 DEBUG [master/7b96b36745c6:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-11-21T19:48:54,576 DEBUG [master/7b96b36745c6:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-11-21T19:48:54,614 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-11-21T19:48:54,643 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-11-21T19:48:54,647 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T19:48:54,649 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-21T19:48:54,649 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-11-21T19:48:54,655 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-11-21T19:48:54,655 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T19:48:54,655 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-21T19:48:54,656 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-11-21T19:48:54,662 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-11-21T19:48:54,662 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T19:48:54,663 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-21T19:48:54,663 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-11-21T19:48:54,668 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-11-21T19:48:54,668 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T19:48:54,669 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-21T19:48:54,670 DEBUG [master/7b96b36745c6:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682 2024-11-21T19:48:54,674 DEBUG [master/7b96b36745c6:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0b774921-3780-c936-4fe1-1d1dcb4ce861/545083c6-d774-d2ca-7409-0fbc2c22cf1a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-21T19:48:54,675 DEBUG [master/7b96b36745c6:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0b774921-3780-c936-4fe1-1d1dcb4ce861/545083c6-d774-d2ca-7409-0fbc2c22cf1a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-21T19:48:54,679 DEBUG [master/7b96b36745c6:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682 2024-11-21T19:48:54,680 DEBUG [master/7b96b36745c6:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682 2024-11-21T19:48:54,683 DEBUG [master/7b96b36745c6:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-21T19:48:54,686 DEBUG [master/7b96b36745c6:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-11-21T19:48:54,694 DEBUG [master/7b96b36745c6:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0b774921-3780-c936-4fe1-1d1dcb4ce861/545083c6-d774-d2ca-7409-0fbc2c22cf1a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-21T19:48:54,694 INFO [master/7b96b36745c6:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=63769697, jitterRate=-0.0497574657201767}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-21T19:48:54,702 DEBUG [master/7b96b36745c6:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682: Writing region info on filesystem at 1732218534589Initializing all the Stores at 1732218534589Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732218534590 (+1 ms)Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732218534592 (+2 ms)Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732218534592Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732218534592Cleaning up temporary data from old regions at 1732218534680 (+88 ms)Region opened successfully at 1732218534702 (+22 ms) 2024-11-21T19:48:54,703 INFO [master/7b96b36745c6:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-11-21T19:48:54,735 DEBUG [master/7b96b36745c6:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@528cbb22, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=7b96b36745c6/172.17.0.2:0 2024-11-21T19:48:54,769 INFO [master/7b96b36745c6:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating... 2024-11-21T19:48:54,782 INFO [master/7b96b36745c6:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5 2024-11-21T19:48:54,782 INFO [master/7b96b36745c6:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50 2024-11-21T19:48:54,785 INFO [master/7b96b36745c6:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-21T19:48:54,786 INFO [master/7b96b36745c6:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-21T19:48:54,793 INFO [master/7b96b36745c6:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 6 msec 2024-11-21T19:48:54,793 INFO [master/7b96b36745c6:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-11-21T19:48:54,820 INFO [master/7b96b36745c6:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2024-11-21T19:48:54,830 DEBUG [master/7b96b36745c6:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:39849-0x10116d829200000, quorum=127.0.0.1:50077, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error) 2024-11-21T19:48:54,832 DEBUG [master/7b96b36745c6:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false 2024-11-21T19:48:54,834 INFO [master/7b96b36745c6:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2024-11-21T19:48:54,836 DEBUG [master/7b96b36745c6:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:39849-0x10116d829200000, quorum=127.0.0.1:50077, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error) 2024-11-21T19:48:54,838 DEBUG [master/7b96b36745c6:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false 2024-11-21T19:48:54,840 INFO [master/7b96b36745c6:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2024-11-21T19:48:54,843 DEBUG [master/7b96b36745c6:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:39849-0x10116d829200000, quorum=127.0.0.1:50077, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error) 2024-11-21T19:48:54,845 DEBUG [master/7b96b36745c6:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false 2024-11-21T19:48:54,846 DEBUG [master/7b96b36745c6:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:39849-0x10116d829200000, quorum=127.0.0.1:50077, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error) 2024-11-21T19:48:54,848 DEBUG [master/7b96b36745c6:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false 2024-11-21T19:48:54,866 DEBUG [master/7b96b36745c6:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:39849-0x10116d829200000, quorum=127.0.0.1:50077, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error) 2024-11-21T19:48:54,868 DEBUG [master/7b96b36745c6:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false 2024-11-21T19:48:54,882 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39849-0x10116d829200000, quorum=127.0.0.1:50077, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-21T19:48:54,882 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43053-0x10116d829200002, quorum=127.0.0.1:50077, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-21T19:48:54,882 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33427-0x10116d829200001, quorum=127.0.0.1:50077, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-21T19:48:54,882 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39849-0x10116d829200000, quorum=127.0.0.1:50077, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-21T19:48:54,882 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33427-0x10116d829200001, quorum=127.0.0.1:50077, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-21T19:48:54,882 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43053-0x10116d829200002, quorum=127.0.0.1:50077, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-21T19:48:54,885 INFO [master/7b96b36745c6:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=7b96b36745c6,39849,1732218533264, sessionid=0x10116d829200000, setting cluster-up flag (Was=false) 2024-11-21T19:48:54,897 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43053-0x10116d829200002, quorum=127.0.0.1:50077, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-21T19:48:54,897 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39849-0x10116d829200000, quorum=127.0.0.1:50077, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-21T19:48:54,897 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33427-0x10116d829200001, quorum=127.0.0.1:50077, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-21T19:48:54,903 DEBUG [master/7b96b36745c6:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort 2024-11-21T19:48:54,905 DEBUG [master/7b96b36745c6:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=7b96b36745c6,39849,1732218533264 2024-11-21T19:48:54,909 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33427-0x10116d829200001, quorum=127.0.0.1:50077, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-21T19:48:54,909 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43053-0x10116d829200002, quorum=127.0.0.1:50077, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-21T19:48:54,909 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39849-0x10116d829200000, quorum=127.0.0.1:50077, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-21T19:48:54,915 DEBUG [master/7b96b36745c6:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort 2024-11-21T19:48:54,916 DEBUG [master/7b96b36745c6:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=7b96b36745c6,39849,1732218533264 2024-11-21T19:48:54,921 INFO [master/7b96b36745c6:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0b774921-3780-c936-4fe1-1d1dcb4ce861/545083c6-d774-d2ca-7409-0fbc2c22cf1a/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again 2024-11-21T19:48:54,961 INFO [RS:0;7b96b36745c6:33427 {}] regionserver.HRegionServer(746): ClusterId : 5d6261fd-cc48-4e27-9635-c975d4b37434 2024-11-21T19:48:54,961 INFO [RS:1;7b96b36745c6:43053 {}] regionserver.HRegionServer(746): ClusterId : 5d6261fd-cc48-4e27-9635-c975d4b37434 2024-11-21T19:48:54,964 DEBUG [RS:0;7b96b36745c6:33427 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-21T19:48:54,964 DEBUG [RS:1;7b96b36745c6:43053 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-21T19:48:54,969 DEBUG [RS:0;7b96b36745c6:33427 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-21T19:48:54,969 DEBUG [RS:1;7b96b36745c6:43053 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-21T19:48:54,969 DEBUG [RS:0;7b96b36745c6:33427 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-21T19:48:54,969 DEBUG [RS:1;7b96b36745c6:43053 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-21T19:48:54,973 DEBUG [RS:1;7b96b36745c6:43053 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-21T19:48:54,973 DEBUG [RS:0;7b96b36745c6:33427 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-21T19:48:54,974 DEBUG [RS:1;7b96b36745c6:43053 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@79f25d5e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=7b96b36745c6/172.17.0.2:0 2024-11-21T19:48:54,974 DEBUG [RS:0;7b96b36745c6:33427 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@76c4b2d6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=7b96b36745c6/172.17.0.2:0 2024-11-21T19:48:54,989 DEBUG [RS:1;7b96b36745c6:43053 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:1;7b96b36745c6:43053 2024-11-21T19:48:54,991 DEBUG [RS:0;7b96b36745c6:33427 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;7b96b36745c6:33427 2024-11-21T19:48:54,991 INFO [RS:1;7b96b36745c6:43053 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-21T19:48:54,991 INFO [RS:0;7b96b36745c6:33427 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-21T19:48:54,992 INFO [RS:0;7b96b36745c6:33427 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-21T19:48:54,992 INFO [RS:1;7b96b36745c6:43053 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-21T19:48:54,992 DEBUG [RS:0;7b96b36745c6:33427 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-21T19:48:54,992 DEBUG [RS:1;7b96b36745c6:43053 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-21T19:48:54,994 INFO [RS:0;7b96b36745c6:33427 {}] regionserver.HRegionServer(2659): reportForDuty to master=7b96b36745c6,39849,1732218533264 with port=33427, startcode=1732218534004 2024-11-21T19:48:54,995 INFO [RS:1;7b96b36745c6:43053 {}] regionserver.HRegionServer(2659): reportForDuty to master=7b96b36745c6,39849,1732218533264 with port=43053, startcode=1732218534126 2024-11-21T19:48:55,001 DEBUG [master/7b96b36745c6:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-21T19:48:55,007 DEBUG [RS:0;7b96b36745c6:33427 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-21T19:48:55,007 DEBUG [RS:1;7b96b36745c6:43053 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-21T19:48:55,014 INFO [master/7b96b36745c6:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2 2024-11-21T19:48:55,022 INFO [master/7b96b36745c6:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2024-11-21T19:48:55,027 DEBUG [master/7b96b36745c6:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: 7b96b36745c6,39849,1732218533264 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2024-11-21T19:48:55,035 DEBUG [master/7b96b36745c6:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/7b96b36745c6:0, corePoolSize=5, maxPoolSize=5 2024-11-21T19:48:55,036 DEBUG [master/7b96b36745c6:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/7b96b36745c6:0, corePoolSize=5, maxPoolSize=5 2024-11-21T19:48:55,036 DEBUG [master/7b96b36745c6:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/7b96b36745c6:0, corePoolSize=5, maxPoolSize=5 2024-11-21T19:48:55,037 DEBUG [master/7b96b36745c6:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/7b96b36745c6:0, corePoolSize=5, maxPoolSize=5 2024-11-21T19:48:55,037 DEBUG [master/7b96b36745c6:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/7b96b36745c6:0, corePoolSize=10, maxPoolSize=10 2024-11-21T19:48:55,037 DEBUG [master/7b96b36745c6:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/7b96b36745c6:0, corePoolSize=1, maxPoolSize=1 2024-11-21T19:48:55,038 DEBUG [master/7b96b36745c6:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/7b96b36745c6:0, corePoolSize=2, maxPoolSize=2 2024-11-21T19:48:55,038 DEBUG [master/7b96b36745c6:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/7b96b36745c6:0, corePoolSize=1, maxPoolSize=1 2024-11-21T19:48:55,046 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-21T19:48:55,047 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region 2024-11-21T19:48:55,050 INFO [master/7b96b36745c6:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732218565050 2024-11-21T19:48:55,052 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T19:48:55,052 INFO [master/7b96b36745c6:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2024-11-21T19:48:55,052 INFO [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-11-21T19:48:55,053 INFO [master/7b96b36745c6:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2024-11-21T19:48:55,057 INFO [master/7b96b36745c6:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2024-11-21T19:48:55,060 INFO [master/7b96b36745c6:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2024-11-21T19:48:55,062 INFO [master/7b96b36745c6:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2024-11-21T19:48:55,062 INFO [master/7b96b36745c6:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2024-11-21T19:48:55,069 INFO [master/7b96b36745c6:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-21T19:48:55,077 INFO [master/7b96b36745c6:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2024-11-21T19:48:55,079 INFO [master/7b96b36745c6:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2024-11-21T19:48:55,079 INFO [master/7b96b36745c6:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2024-11-21T19:48:55,083 INFO [master/7b96b36745c6:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2024-11-21T19:48:55,083 INFO [master/7b96b36745c6:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2024-11-21T19:48:55,085 DEBUG [master/7b96b36745c6:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/7b96b36745c6:0:becomeActiveMaster-HFileCleaner.large.0-1732218535084,5,FailOnTimeoutGroup] 2024-11-21T19:48:55,087 DEBUG [master/7b96b36745c6:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/7b96b36745c6:0:becomeActiveMaster-HFileCleaner.small.0-1732218535085,5,FailOnTimeoutGroup] 2024-11-21T19:48:55,088 INFO [master/7b96b36745c6:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-21T19:48:55,088 INFO [master/7b96b36745c6:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2024-11-21T19:48:55,094 INFO [master/7b96b36745c6:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2024-11-21T19:48:55,094 INFO [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0b774921-3780-c936-4fe1-1d1dcb4ce861/545083c6-d774-d2ca-7409-0fbc2c22cf1a/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321 2024-11-21T19:48:55,094 INFO [master/7b96b36745c6:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2024-11-21T19:48:55,095 INFO [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0b774921-3780-c936-4fe1-1d1dcb4ce861/545083c6-d774-d2ca-7409-0fbc2c22cf1a 2024-11-21T19:48:55,096 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52281, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=RegionServerStatusService 2024-11-21T19:48:55,096 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53529, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=RegionServerStatusService 2024-11-21T19:48:55,104 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=39849 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 7b96b36745c6,33427,1732218534004 2024-11-21T19:48:55,108 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=39849 {}] master.ServerManager(517): Registering regionserver=7b96b36745c6,33427,1732218534004 2024-11-21T19:48:55,110 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-21T19:48:55,113 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family info of region 1588230740 2024-11-21T19:48:55,120 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=39849 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 7b96b36745c6,43053,1732218534126 2024-11-21T19:48:55,121 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=39849 {}] master.ServerManager(517): Registering regionserver=7b96b36745c6,43053,1732218534126 2024-11-21T19:48:55,125 DEBUG [RS:0;7b96b36745c6:33427 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0b774921-3780-c936-4fe1-1d1dcb4ce861/545083c6-d774-d2ca-7409-0fbc2c22cf1a 2024-11-21T19:48:55,125 DEBUG [RS:0;7b96b36745c6:33427 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=file:/ 2024-11-21T19:48:55,125 DEBUG [RS:0;7b96b36745c6:33427 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-21T19:48:55,125 DEBUG [RS:1;7b96b36745c6:43053 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0b774921-3780-c936-4fe1-1d1dcb4ce861/545083c6-d774-d2ca-7409-0fbc2c22cf1a 2024-11-21T19:48:55,126 DEBUG [RS:1;7b96b36745c6:43053 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=file:/ 2024-11-21T19:48:55,126 DEBUG [RS:1;7b96b36745c6:43053 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-21T19:48:55,127 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-21T19:48:55,127 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T19:48:55,128 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-21T19:48:55,128 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family ns of region 1588230740 2024-11-21T19:48:55,133 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39849-0x10116d829200000, quorum=127.0.0.1:50077, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-21T19:48:55,134 DEBUG [RS:1;7b96b36745c6:43053 {}] zookeeper.ZKUtil(111): regionserver:43053-0x10116d829200002, quorum=127.0.0.1:50077, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/7b96b36745c6,43053,1732218534126 2024-11-21T19:48:55,134 WARN [RS:1;7b96b36745c6:43053 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-21T19:48:55,134 DEBUG [RS:0;7b96b36745c6:33427 {}] zookeeper.ZKUtil(111): regionserver:33427-0x10116d829200001, quorum=127.0.0.1:50077, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/7b96b36745c6,33427,1732218534004 2024-11-21T19:48:55,134 WARN [RS:0;7b96b36745c6:33427 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-21T19:48:55,134 INFO [RS:1;7b96b36745c6:43053 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-21T19:48:55,135 INFO [RS:0;7b96b36745c6:33427 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-21T19:48:55,135 DEBUG [RS:1;7b96b36745c6:43053 {}] regionserver.HRegionServer(1793): logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0b774921-3780-c936-4fe1-1d1dcb4ce861/545083c6-d774-d2ca-7409-0fbc2c22cf1a/WALs/7b96b36745c6,43053,1732218534126 2024-11-21T19:48:55,135 DEBUG [RS:0;7b96b36745c6:33427 {}] regionserver.HRegionServer(1793): logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0b774921-3780-c936-4fe1-1d1dcb4ce861/545083c6-d774-d2ca-7409-0fbc2c22cf1a/WALs/7b96b36745c6,33427,1732218534004 2024-11-21T19:48:55,138 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-21T19:48:55,139 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T19:48:55,139 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-21T19:48:55,139 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family rep_barrier of region 1588230740 2024-11-21T19:48:55,141 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [7b96b36745c6,33427,1732218534004] 2024-11-21T19:48:55,141 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [7b96b36745c6,43053,1732218534126] 2024-11-21T19:48:55,146 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-21T19:48:55,146 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T19:48:55,146 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-21T19:48:55,147 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family table of region 1588230740 2024-11-21T19:48:55,154 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-21T19:48:55,154 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T19:48:55,154 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-21T19:48:55,155 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-21T19:48:55,156 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0b774921-3780-c936-4fe1-1d1dcb4ce861/545083c6-d774-d2ca-7409-0fbc2c22cf1a/data/hbase/meta/1588230740 2024-11-21T19:48:55,156 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0b774921-3780-c936-4fe1-1d1dcb4ce861/545083c6-d774-d2ca-7409-0fbc2c22cf1a/data/hbase/meta/1588230740 2024-11-21T19:48:55,156 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-21T19:48:55,156 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-21T19:48:55,157 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-21T19:48:55,158 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-21T19:48:55,166 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0b774921-3780-c936-4fe1-1d1dcb4ce861/545083c6-d774-d2ca-7409-0fbc2c22cf1a/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-21T19:48:55,166 INFO [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=58877080, jitterRate=-0.12266314029693604}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-21T19:48:55,170 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740: Writing region info on filesystem at 1732218535110Initializing all the Stores at 1732218535110Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732218535110Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732218535113 (+3 ms)Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732218535113Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732218535113Cleaning up temporary data from old regions at 1732218535156 (+43 ms)Region opened successfully at 1732218535170 (+14 ms) 2024-11-21T19:48:55,170 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-21T19:48:55,171 INFO [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-21T19:48:55,171 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-21T19:48:55,171 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-21T19:48:55,171 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-21T19:48:55,173 INFO [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-21T19:48:55,173 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1732218535170Disabling compacts and flushes for region at 1732218535170Disabling writes for close at 1732218535171 (+1 ms)Writing region close event to WAL at 1732218535173 (+2 ms)Closed at 1732218535173 2024-11-21T19:48:55,175 INFO [RS:0;7b96b36745c6:33427 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-21T19:48:55,175 INFO [RS:1;7b96b36745c6:43053 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-21T19:48:55,175 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-21T19:48:55,176 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta 2024-11-21T19:48:55,184 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-11-21T19:48:55,193 INFO [RS:0;7b96b36745c6:33427 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-21T19:48:55,194 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-21T19:48:55,195 INFO [RS:1;7b96b36745c6:43053 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-21T19:48:55,197 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2024-11-21T19:48:55,202 INFO [RS:1;7b96b36745c6:43053 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-21T19:48:55,202 INFO [RS:0;7b96b36745c6:33427 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-21T19:48:55,202 INFO [RS:0;7b96b36745c6:33427 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-21T19:48:55,202 INFO [RS:1;7b96b36745c6:43053 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-21T19:48:55,203 INFO [RS:1;7b96b36745c6:43053 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-21T19:48:55,203 INFO [RS:0;7b96b36745c6:33427 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-21T19:48:55,208 INFO [RS:0;7b96b36745c6:33427 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-21T19:48:55,208 INFO [RS:1;7b96b36745c6:43053 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-21T19:48:55,210 INFO [RS:1;7b96b36745c6:43053 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-21T19:48:55,210 INFO [RS:0;7b96b36745c6:33427 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-21T19:48:55,210 DEBUG [RS:0;7b96b36745c6:33427 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/7b96b36745c6:0, corePoolSize=1, maxPoolSize=1 2024-11-21T19:48:55,210 DEBUG [RS:1;7b96b36745c6:43053 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/7b96b36745c6:0, corePoolSize=1, maxPoolSize=1 2024-11-21T19:48:55,211 DEBUG [RS:0;7b96b36745c6:33427 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/7b96b36745c6:0, corePoolSize=1, maxPoolSize=1 2024-11-21T19:48:55,211 DEBUG [RS:1;7b96b36745c6:43053 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/7b96b36745c6:0, corePoolSize=1, maxPoolSize=1 2024-11-21T19:48:55,211 DEBUG [RS:1;7b96b36745c6:43053 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/7b96b36745c6:0, corePoolSize=1, maxPoolSize=1 2024-11-21T19:48:55,211 DEBUG [RS:0;7b96b36745c6:33427 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/7b96b36745c6:0, corePoolSize=1, maxPoolSize=1 2024-11-21T19:48:55,211 DEBUG [RS:1;7b96b36745c6:43053 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/7b96b36745c6:0, corePoolSize=1, maxPoolSize=1 2024-11-21T19:48:55,211 DEBUG [RS:0;7b96b36745c6:33427 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/7b96b36745c6:0, corePoolSize=1, maxPoolSize=1 2024-11-21T19:48:55,211 DEBUG [RS:1;7b96b36745c6:43053 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/7b96b36745c6:0, corePoolSize=1, maxPoolSize=1 2024-11-21T19:48:55,211 DEBUG [RS:0;7b96b36745c6:33427 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/7b96b36745c6:0, corePoolSize=1, maxPoolSize=1 2024-11-21T19:48:55,212 DEBUG [RS:1;7b96b36745c6:43053 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/7b96b36745c6:0, corePoolSize=2, maxPoolSize=2 2024-11-21T19:48:55,212 DEBUG [RS:0;7b96b36745c6:33427 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/7b96b36745c6:0, corePoolSize=2, maxPoolSize=2 2024-11-21T19:48:55,212 DEBUG [RS:1;7b96b36745c6:43053 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/7b96b36745c6:0, corePoolSize=1, maxPoolSize=1 2024-11-21T19:48:55,212 DEBUG [RS:0;7b96b36745c6:33427 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/7b96b36745c6:0, corePoolSize=1, maxPoolSize=1 2024-11-21T19:48:55,212 DEBUG [RS:1;7b96b36745c6:43053 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/7b96b36745c6:0, corePoolSize=1, maxPoolSize=1 2024-11-21T19:48:55,212 DEBUG [RS:0;7b96b36745c6:33427 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/7b96b36745c6:0, corePoolSize=1, maxPoolSize=1 2024-11-21T19:48:55,212 DEBUG [RS:1;7b96b36745c6:43053 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/7b96b36745c6:0, corePoolSize=1, maxPoolSize=1 2024-11-21T19:48:55,212 DEBUG [RS:0;7b96b36745c6:33427 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/7b96b36745c6:0, corePoolSize=1, maxPoolSize=1 2024-11-21T19:48:55,212 DEBUG [RS:1;7b96b36745c6:43053 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/7b96b36745c6:0, corePoolSize=1, maxPoolSize=1 2024-11-21T19:48:55,212 DEBUG [RS:0;7b96b36745c6:33427 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/7b96b36745c6:0, corePoolSize=1, maxPoolSize=1 2024-11-21T19:48:55,213 DEBUG [RS:1;7b96b36745c6:43053 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/7b96b36745c6:0, corePoolSize=1, maxPoolSize=1 2024-11-21T19:48:55,213 DEBUG [RS:0;7b96b36745c6:33427 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/7b96b36745c6:0, corePoolSize=1, maxPoolSize=1 2024-11-21T19:48:55,213 DEBUG [RS:0;7b96b36745c6:33427 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/7b96b36745c6:0, corePoolSize=1, maxPoolSize=1 2024-11-21T19:48:55,213 DEBUG [RS:1;7b96b36745c6:43053 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/7b96b36745c6:0, corePoolSize=1, maxPoolSize=1 2024-11-21T19:48:55,213 DEBUG [RS:0;7b96b36745c6:33427 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/7b96b36745c6:0, corePoolSize=3, maxPoolSize=3 2024-11-21T19:48:55,213 DEBUG [RS:1;7b96b36745c6:43053 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/7b96b36745c6:0, corePoolSize=3, maxPoolSize=3 2024-11-21T19:48:55,213 DEBUG [RS:1;7b96b36745c6:43053 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/7b96b36745c6:0, corePoolSize=3, maxPoolSize=3 2024-11-21T19:48:55,213 DEBUG [RS:0;7b96b36745c6:33427 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/7b96b36745c6:0, corePoolSize=3, maxPoolSize=3 2024-11-21T19:48:55,217 INFO [RS:1;7b96b36745c6:43053 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-21T19:48:55,217 INFO [RS:1;7b96b36745c6:43053 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-21T19:48:55,218 INFO [RS:1;7b96b36745c6:43053 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-21T19:48:55,218 INFO [RS:1;7b96b36745c6:43053 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-21T19:48:55,218 INFO [RS:1;7b96b36745c6:43053 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-21T19:48:55,218 INFO [RS:1;7b96b36745c6:43053 {}] hbase.ChoreService(168): Chore ScheduledChore name=7b96b36745c6,43053,1732218534126-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-21T19:48:55,220 INFO [RS:0;7b96b36745c6:33427 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-21T19:48:55,220 INFO [RS:0;7b96b36745c6:33427 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-21T19:48:55,221 INFO [RS:0;7b96b36745c6:33427 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-21T19:48:55,221 INFO [RS:0;7b96b36745c6:33427 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-21T19:48:55,221 INFO [RS:0;7b96b36745c6:33427 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-21T19:48:55,221 INFO [RS:0;7b96b36745c6:33427 {}] hbase.ChoreService(168): Chore ScheduledChore name=7b96b36745c6,33427,1732218534004-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-21T19:48:55,246 INFO [RS:1;7b96b36745c6:43053 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-21T19:48:55,246 INFO [RS:0;7b96b36745c6:33427 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-21T19:48:55,249 INFO [RS:0;7b96b36745c6:33427 {}] hbase.ChoreService(168): Chore ScheduledChore name=7b96b36745c6,33427,1732218534004-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-21T19:48:55,249 INFO [RS:1;7b96b36745c6:43053 {}] hbase.ChoreService(168): Chore ScheduledChore name=7b96b36745c6,43053,1732218534126-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-21T19:48:55,250 INFO [RS:1;7b96b36745c6:43053 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-21T19:48:55,250 INFO [RS:0;7b96b36745c6:33427 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-21T19:48:55,250 INFO [RS:1;7b96b36745c6:43053 {}] regionserver.Replication(171): 7b96b36745c6,43053,1732218534126 started 2024-11-21T19:48:55,250 INFO [RS:0;7b96b36745c6:33427 {}] regionserver.Replication(171): 7b96b36745c6,33427,1732218534004 started 2024-11-21T19:48:55,278 INFO [RS:0;7b96b36745c6:33427 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-21T19:48:55,278 INFO [RS:1;7b96b36745c6:43053 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-21T19:48:55,278 INFO [RS:0;7b96b36745c6:33427 {}] regionserver.HRegionServer(1482): Serving as 7b96b36745c6,33427,1732218534004, RpcServer on 7b96b36745c6/172.17.0.2:33427, sessionid=0x10116d829200001 2024-11-21T19:48:55,278 INFO [RS:1;7b96b36745c6:43053 {}] regionserver.HRegionServer(1482): Serving as 7b96b36745c6,43053,1732218534126, RpcServer on 7b96b36745c6/172.17.0.2:43053, sessionid=0x10116d829200002 2024-11-21T19:48:55,279 DEBUG [RS:0;7b96b36745c6:33427 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-21T19:48:55,279 DEBUG [RS:1;7b96b36745c6:43053 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-21T19:48:55,279 DEBUG [RS:0;7b96b36745c6:33427 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 7b96b36745c6,33427,1732218534004 2024-11-21T19:48:55,279 DEBUG [RS:1;7b96b36745c6:43053 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 7b96b36745c6,43053,1732218534126 2024-11-21T19:48:55,280 DEBUG [RS:0;7b96b36745c6:33427 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '7b96b36745c6,33427,1732218534004' 2024-11-21T19:48:55,280 DEBUG [RS:1;7b96b36745c6:43053 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '7b96b36745c6,43053,1732218534126' 2024-11-21T19:48:55,280 DEBUG [RS:0;7b96b36745c6:33427 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-21T19:48:55,280 DEBUG [RS:1;7b96b36745c6:43053 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-21T19:48:55,281 DEBUG [RS:1;7b96b36745c6:43053 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-21T19:48:55,281 DEBUG [RS:0;7b96b36745c6:33427 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-21T19:48:55,282 DEBUG [RS:1;7b96b36745c6:43053 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-21T19:48:55,282 DEBUG [RS:0;7b96b36745c6:33427 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-21T19:48:55,282 DEBUG [RS:1;7b96b36745c6:43053 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-21T19:48:55,282 DEBUG [RS:0;7b96b36745c6:33427 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-21T19:48:55,282 DEBUG [RS:0;7b96b36745c6:33427 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 7b96b36745c6,33427,1732218534004 2024-11-21T19:48:55,282 DEBUG [RS:1;7b96b36745c6:43053 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 7b96b36745c6,43053,1732218534126 2024-11-21T19:48:55,282 DEBUG [RS:1;7b96b36745c6:43053 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '7b96b36745c6,43053,1732218534126' 2024-11-21T19:48:55,282 DEBUG [RS:0;7b96b36745c6:33427 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '7b96b36745c6,33427,1732218534004' 2024-11-21T19:48:55,282 DEBUG [RS:1;7b96b36745c6:43053 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-21T19:48:55,282 DEBUG [RS:0;7b96b36745c6:33427 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-21T19:48:55,283 DEBUG [RS:0;7b96b36745c6:33427 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-21T19:48:55,283 DEBUG [RS:1;7b96b36745c6:43053 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-21T19:48:55,284 DEBUG [RS:1;7b96b36745c6:43053 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-21T19:48:55,284 INFO [RS:1;7b96b36745c6:43053 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-21T19:48:55,284 INFO [RS:1;7b96b36745c6:43053 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-21T19:48:55,284 DEBUG [RS:0;7b96b36745c6:33427 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-21T19:48:55,285 INFO [RS:0;7b96b36745c6:33427 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-21T19:48:55,285 INFO [RS:0;7b96b36745c6:33427 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-21T19:48:55,348 WARN [7b96b36745c6:39849 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-21T19:48:55,394 INFO [RS:0;7b96b36745c6:33427 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-21T19:48:55,394 INFO [RS:1;7b96b36745c6:43053 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-21T19:48:55,395 INFO [RS:1;7b96b36745c6:43053 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=7b96b36745c6%2C43053%2C1732218534126, suffix=, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0b774921-3780-c936-4fe1-1d1dcb4ce861/545083c6-d774-d2ca-7409-0fbc2c22cf1a/WALs/7b96b36745c6,43053,1732218534126, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0b774921-3780-c936-4fe1-1d1dcb4ce861/545083c6-d774-d2ca-7409-0fbc2c22cf1a/oldWALs, maxLogs=55 2024-11-21T19:48:55,396 INFO [RS:0;7b96b36745c6:33427 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=7b96b36745c6%2C33427%2C1732218534004, suffix=, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0b774921-3780-c936-4fe1-1d1dcb4ce861/545083c6-d774-d2ca-7409-0fbc2c22cf1a/WALs/7b96b36745c6,33427,1732218534004, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0b774921-3780-c936-4fe1-1d1dcb4ce861/545083c6-d774-d2ca-7409-0fbc2c22cf1a/oldWALs, maxLogs=55 2024-11-21T19:48:55,422 INFO [RS:0;7b96b36745c6:33427 {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0b774921-3780-c936-4fe1-1d1dcb4ce861/545083c6-d774-d2ca-7409-0fbc2c22cf1a/WALs/7b96b36745c6,33427,1732218534004/7b96b36745c6%2C33427%2C1732218534004.1732218535401 2024-11-21T19:48:55,424 DEBUG [RS:0;7b96b36745c6:33427 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-11-21T19:48:55,427 INFO [RS:1;7b96b36745c6:43053 {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0b774921-3780-c936-4fe1-1d1dcb4ce861/545083c6-d774-d2ca-7409-0fbc2c22cf1a/WALs/7b96b36745c6,43053,1732218534126/7b96b36745c6%2C43053%2C1732218534126.1732218535401 2024-11-21T19:48:55,428 DEBUG [RS:1;7b96b36745c6:43053 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-11-21T19:48:55,601 DEBUG [7b96b36745c6:39849 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=2, allServersCount=2 2024-11-21T19:48:55,612 DEBUG [7b96b36745c6:39849 {}] balancer.BalancerClusterState(204): Hosts are {7b96b36745c6=0} racks are {/default-rack=0} 2024-11-21T19:48:55,617 DEBUG [7b96b36745c6:39849 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-11-21T19:48:55,617 DEBUG [7b96b36745c6:39849 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-11-21T19:48:55,617 DEBUG [7b96b36745c6:39849 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-11-21T19:48:55,617 DEBUG [7b96b36745c6:39849 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-11-21T19:48:55,617 INFO [7b96b36745c6:39849 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-11-21T19:48:55,617 INFO [7b96b36745c6:39849 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-11-21T19:48:55,617 DEBUG [7b96b36745c6:39849 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-11-21T19:48:55,625 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=7b96b36745c6,43053,1732218534126 2024-11-21T19:48:55,632 INFO [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 7b96b36745c6,43053,1732218534126, state=OPENING 2024-11-21T19:48:55,638 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2024-11-21T19:48:55,640 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43053-0x10116d829200002, quorum=127.0.0.1:50077, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-21T19:48:55,640 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39849-0x10116d829200000, quorum=127.0.0.1:50077, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-21T19:48:55,640 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33427-0x10116d829200001, quorum=127.0.0.1:50077, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-21T19:48:55,641 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-21T19:48:55,641 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-21T19:48:55,641 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-21T19:48:55,643 DEBUG [PEWorker-3 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-21T19:48:55,645 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=7b96b36745c6,43053,1732218534126}] 2024-11-21T19:48:55,820 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-21T19:48:55,823 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-4-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54885, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-21T19:48:55,832 INFO [RS_OPEN_META-regionserver/7b96b36745c6:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-11-21T19:48:55,832 INFO [RS_OPEN_META-regionserver/7b96b36745c6:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-21T19:48:55,833 INFO [RS_OPEN_META-regionserver/7b96b36745c6:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-11-21T19:48:55,834 INFO [RS_OPEN_META-regionserver/7b96b36745c6:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=7b96b36745c6%2C43053%2C1732218534126.meta, suffix=.meta, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0b774921-3780-c936-4fe1-1d1dcb4ce861/545083c6-d774-d2ca-7409-0fbc2c22cf1a/WALs/7b96b36745c6,43053,1732218534126, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0b774921-3780-c936-4fe1-1d1dcb4ce861/545083c6-d774-d2ca-7409-0fbc2c22cf1a/oldWALs, maxLogs=55 2024-11-21T19:48:55,855 INFO [RS_OPEN_META-regionserver/7b96b36745c6:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0b774921-3780-c936-4fe1-1d1dcb4ce861/545083c6-d774-d2ca-7409-0fbc2c22cf1a/WALs/7b96b36745c6,43053,1732218534126/7b96b36745c6%2C43053%2C1732218534126.meta.1732218535836.meta 2024-11-21T19:48:55,857 DEBUG [RS_OPEN_META-regionserver/7b96b36745c6:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-11-21T19:48:55,857 DEBUG [RS_OPEN_META-regionserver/7b96b36745c6:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-11-21T19:48:55,860 DEBUG [RS_OPEN_META-regionserver/7b96b36745c6:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-11-21T19:48:55,863 DEBUG [RS_OPEN_META-regionserver/7b96b36745c6:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-11-21T19:48:55,869 INFO [RS_OPEN_META-regionserver/7b96b36745c6:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-11-21T19:48:55,873 DEBUG [RS_OPEN_META-regionserver/7b96b36745c6:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-11-21T19:48:55,874 DEBUG [RS_OPEN_META-regionserver/7b96b36745c6:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-21T19:48:55,874 DEBUG [RS_OPEN_META-regionserver/7b96b36745c6:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-11-21T19:48:55,874 DEBUG [RS_OPEN_META-regionserver/7b96b36745c6:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-11-21T19:48:55,876 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family info of region 1588230740 2024-11-21T19:48:55,877 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-21T19:48:55,877 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T19:48:55,878 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-21T19:48:55,878 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family ns of region 1588230740 2024-11-21T19:48:55,879 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-21T19:48:55,879 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T19:48:55,880 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-21T19:48:55,880 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family rep_barrier of region 1588230740 2024-11-21T19:48:55,881 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-21T19:48:55,881 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T19:48:55,882 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-21T19:48:55,882 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family table of region 1588230740 2024-11-21T19:48:55,883 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-21T19:48:55,883 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T19:48:55,883 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-21T19:48:55,884 DEBUG [RS_OPEN_META-regionserver/7b96b36745c6:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-21T19:48:55,885 DEBUG [RS_OPEN_META-regionserver/7b96b36745c6:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0b774921-3780-c936-4fe1-1d1dcb4ce861/545083c6-d774-d2ca-7409-0fbc2c22cf1a/data/hbase/meta/1588230740 2024-11-21T19:48:55,885 DEBUG [RS_OPEN_META-regionserver/7b96b36745c6:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0b774921-3780-c936-4fe1-1d1dcb4ce861/545083c6-d774-d2ca-7409-0fbc2c22cf1a/data/hbase/meta/1588230740 2024-11-21T19:48:55,886 DEBUG [RS_OPEN_META-regionserver/7b96b36745c6:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-21T19:48:55,886 DEBUG [RS_OPEN_META-regionserver/7b96b36745c6:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-21T19:48:55,886 DEBUG [RS_OPEN_META-regionserver/7b96b36745c6:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-21T19:48:55,887 DEBUG [RS_OPEN_META-regionserver/7b96b36745c6:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-21T19:48:55,888 INFO [RS_OPEN_META-regionserver/7b96b36745c6:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=70658519, jitterRate=0.0528939813375473}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-21T19:48:55,888 DEBUG [RS_OPEN_META-regionserver/7b96b36745c6:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-11-21T19:48:55,890 DEBUG [RS_OPEN_META-regionserver/7b96b36745c6:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1732218535875Writing region info on filesystem at 1732218535875Initializing all the Stores at 1732218535875Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732218535875Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732218535876 (+1 ms)Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732218535876Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732218535876Cleaning up temporary data from old regions at 1732218535886 (+10 ms)Running coprocessor post-open hooks at 1732218535888 (+2 ms)Region opened successfully at 1732218535890 (+2 ms) 2024-11-21T19:48:55,896 INFO [RS_OPEN_META-regionserver/7b96b36745c6:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1732218535811 2024-11-21T19:48:55,908 DEBUG [RS_OPEN_META-regionserver/7b96b36745c6:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-11-21T19:48:55,908 INFO [RS_OPEN_META-regionserver/7b96b36745c6:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-11-21T19:48:55,909 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=7b96b36745c6,43053,1732218534126 2024-11-21T19:48:55,911 INFO [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 7b96b36745c6,43053,1732218534126, state=OPEN 2024-11-21T19:48:55,914 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39849-0x10116d829200000, quorum=127.0.0.1:50077, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-21T19:48:55,914 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33427-0x10116d829200001, quorum=127.0.0.1:50077, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-21T19:48:55,914 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43053-0x10116d829200002, quorum=127.0.0.1:50077, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-21T19:48:55,914 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-21T19:48:55,914 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-21T19:48:55,914 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-21T19:48:55,915 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=7b96b36745c6,43053,1732218534126 2024-11-21T19:48:55,920 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2 2024-11-21T19:48:55,920 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=7b96b36745c6,43053,1732218534126 in 270 msec 2024-11-21T19:48:55,926 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1 2024-11-21T19:48:55,926 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 738 msec 2024-11-21T19:48:55,927 DEBUG [PEWorker-2 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-21T19:48:55,927 INFO [PEWorker-2 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces 2024-11-21T19:48:55,951 DEBUG [PEWorker-2 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-21T19:48:55,953 DEBUG [PEWorker-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=7b96b36745c6,43053,1732218534126, seqNum=-1] 2024-11-21T19:48:55,978 DEBUG [PEWorker-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-21T19:48:55,980 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-4-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:32785, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-21T19:48:56,002 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 1.0520 sec 2024-11-21T19:48:56,003 INFO [master/7b96b36745c6:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1732218536003, completionTime=-1 2024-11-21T19:48:56,008 INFO [master/7b96b36745c6:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=2; waited=0ms, expected min=2 server(s), max=2 server(s), master is running 2024-11-21T19:48:56,008 DEBUG [master/7b96b36745c6:0:becomeActiveMaster {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-21T19:48:56,040 INFO [master/7b96b36745c6:0:becomeActiveMaster {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-11-21T19:48:56,040 INFO [master/7b96b36745c6:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732218596040 2024-11-21T19:48:56,041 INFO [master/7b96b36745c6:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732218656040 2024-11-21T19:48:56,041 INFO [master/7b96b36745c6:0:becomeActiveMaster {}] assignment.AssignmentManager(1783): Joined the cluster in 32 msec 2024-11-21T19:48:56,043 DEBUG [master/7b96b36745c6:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 1588230740 changed from -1.0 to 0.0, refreshing cache 2024-11-21T19:48:56,052 INFO [master/7b96b36745c6:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=7b96b36745c6,39849,1732218533264-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-21T19:48:56,052 INFO [master/7b96b36745c6:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=7b96b36745c6,39849,1732218533264-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-21T19:48:56,052 INFO [master/7b96b36745c6:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=7b96b36745c6,39849,1732218533264-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-21T19:48:56,055 INFO [master/7b96b36745c6:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-7b96b36745c6:39849, period=300000, unit=MILLISECONDS is enabled. 2024-11-21T19:48:56,056 INFO [master/7b96b36745c6:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2024-11-21T19:48:56,058 INFO [master/7b96b36745c6:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled. 2024-11-21T19:48:56,063 DEBUG [master/7b96b36745c6:0.Chore.1 {}] janitor.CatalogJanitor(180): 2024-11-21T19:48:56,099 INFO [master/7b96b36745c6:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 1.877sec 2024-11-21T19:48:56,107 INFO [master/7b96b36745c6:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2024-11-21T19:48:56,108 INFO [master/7b96b36745c6:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2024-11-21T19:48:56,109 INFO [master/7b96b36745c6:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2024-11-21T19:48:56,110 INFO [master/7b96b36745c6:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2024-11-21T19:48:56,110 INFO [master/7b96b36745c6:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2024-11-21T19:48:56,111 INFO [master/7b96b36745c6:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=7b96b36745c6,39849,1732218533264-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-21T19:48:56,112 INFO [master/7b96b36745c6:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=7b96b36745c6,39849,1732218533264-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2024-11-21T19:48:56,113 DEBUG [master/7b96b36745c6:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds 2024-11-21T19:48:56,114 INFO [master/7b96b36745c6:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2024-11-21T19:48:56,115 INFO [master/7b96b36745c6:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=7b96b36745c6,39849,1732218533264-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-21T19:48:56,179 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@71985066, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T19:48:56,182 DEBUG [Time-limited test {}] nio.NioEventLoop(110): -Dio.netty.noKeySetOptimization: false 2024-11-21T19:48:56,182 DEBUG [Time-limited test {}] nio.NioEventLoop(111): -Dio.netty.selectorAutoRebuildThreshold: 512 2024-11-21T19:48:56,187 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 7b96b36745c6,39849,-1 for getting cluster id 2024-11-21T19:48:56,190 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-21T19:48:56,199 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '5d6261fd-cc48-4e27-9635-c975d4b37434' 2024-11-21T19:48:56,201 DEBUG [RPCClient-NioEventLoopGroup-5-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-21T19:48:56,201 DEBUG [RPCClient-NioEventLoopGroup-5-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "5d6261fd-cc48-4e27-9635-c975d4b37434" 2024-11-21T19:48:56,202 DEBUG [RPCClient-NioEventLoopGroup-5-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@74265abe, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T19:48:56,202 DEBUG [RPCClient-NioEventLoopGroup-5-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [7b96b36745c6,39849,-1] 2024-11-21T19:48:56,205 DEBUG [RPCClient-NioEventLoopGroup-5-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-21T19:48:56,206 DEBUG [RPCClient-NioEventLoopGroup-5-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T19:48:56,208 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54824, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-21T19:48:56,211 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2e124b8e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T19:48:56,212 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-21T19:48:56,220 DEBUG [RPCClient-NioEventLoopGroup-5-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=7b96b36745c6,43053,1732218534126, seqNum=-1] 2024-11-21T19:48:56,220 DEBUG [RPCClient-NioEventLoopGroup-5-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-21T19:48:56,223 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-4-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35516, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-21T19:48:56,256 INFO [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=7b96b36745c6,39849,1732218533264 2024-11-21T19:48:56,262 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-21T19:48:56,267 DEBUG [RPCClient-NioEventLoopGroup-5-2 {}] client.AsyncConnectionImpl(321): The fetched master address is 7b96b36745c6,39849,1732218533264 2024-11-21T19:48:56,270 DEBUG [RPCClient-NioEventLoopGroup-5-2 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@a2d937d 2024-11-21T19:48:56,271 DEBUG [RPCClient-NioEventLoopGroup-5-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-21T19:48:56,277 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54838, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-21T19:48:56,284 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39849 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'testBlockEvictionOnRegionMove1732218536256', {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-21T19:48:56,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39849 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=testBlockEvictionOnRegionMove1732218536256 2024-11-21T19:48:56,293 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=testBlockEvictionOnRegionMove1732218536256 execute state=CREATE_TABLE_PRE_OPERATION 2024-11-21T19:48:56,295 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39849 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "testBlockEvictionOnRegionMove1732218536256" procId is: 4 2024-11-21T19:48:56,296 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T19:48:56,297 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=testBlockEvictionOnRegionMove1732218536256 execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-21T19:48:56,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39849 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-21T19:48:56,318 INFO [RegionOpenAndInit-testBlockEvictionOnRegionMove1732218536256-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 7b850c849b1ef94018414fda0d571719, NAME => 'testBlockEvictionOnRegionMove1732218536256,,1732218536279.7b850c849b1ef94018414fda0d571719.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testBlockEvictionOnRegionMove1732218536256', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0b774921-3780-c936-4fe1-1d1dcb4ce861/545083c6-d774-d2ca-7409-0fbc2c22cf1a 2024-11-21T19:48:56,325 DEBUG [RegionOpenAndInit-testBlockEvictionOnRegionMove1732218536256-pool-0 {}] regionserver.HRegion(898): Instantiated testBlockEvictionOnRegionMove1732218536256,,1732218536279.7b850c849b1ef94018414fda0d571719.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-21T19:48:56,326 DEBUG [RegionOpenAndInit-testBlockEvictionOnRegionMove1732218536256-pool-0 {}] regionserver.HRegion(1722): Closing 7b850c849b1ef94018414fda0d571719, disabling compactions & flushes 2024-11-21T19:48:56,326 INFO [RegionOpenAndInit-testBlockEvictionOnRegionMove1732218536256-pool-0 {}] regionserver.HRegion(1755): Closing region testBlockEvictionOnRegionMove1732218536256,,1732218536279.7b850c849b1ef94018414fda0d571719. 2024-11-21T19:48:56,326 DEBUG [RegionOpenAndInit-testBlockEvictionOnRegionMove1732218536256-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on testBlockEvictionOnRegionMove1732218536256,,1732218536279.7b850c849b1ef94018414fda0d571719. 2024-11-21T19:48:56,326 DEBUG [RegionOpenAndInit-testBlockEvictionOnRegionMove1732218536256-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on testBlockEvictionOnRegionMove1732218536256,,1732218536279.7b850c849b1ef94018414fda0d571719. after waiting 0 ms 2024-11-21T19:48:56,326 DEBUG [RegionOpenAndInit-testBlockEvictionOnRegionMove1732218536256-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region testBlockEvictionOnRegionMove1732218536256,,1732218536279.7b850c849b1ef94018414fda0d571719. 2024-11-21T19:48:56,326 INFO [RegionOpenAndInit-testBlockEvictionOnRegionMove1732218536256-pool-0 {}] regionserver.HRegion(1973): Closed testBlockEvictionOnRegionMove1732218536256,,1732218536279.7b850c849b1ef94018414fda0d571719. 2024-11-21T19:48:56,326 DEBUG [RegionOpenAndInit-testBlockEvictionOnRegionMove1732218536256-pool-0 {}] regionserver.HRegion(1676): Region close journal for 7b850c849b1ef94018414fda0d571719: Waiting for close lock at 1732218536326Disabling compacts and flushes for region at 1732218536326Disabling writes for close at 1732218536326Writing region close event to WAL at 1732218536326Closed at 1732218536326 2024-11-21T19:48:56,328 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=testBlockEvictionOnRegionMove1732218536256 execute state=CREATE_TABLE_ADD_TO_META 2024-11-21T19:48:56,333 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testBlockEvictionOnRegionMove1732218536256,,1732218536279.7b850c849b1ef94018414fda0d571719.","families":{"info":[{"qualifier":"regioninfo","vlen":76,"tag":[],"timestamp":"1732218536328"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732218536328"}]},"ts":"1732218536328"} 2024-11-21T19:48:56,341 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-21T19:48:56,342 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=testBlockEvictionOnRegionMove1732218536256 execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-21T19:48:56,345 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testBlockEvictionOnRegionMove1732218536256","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732218536342"}]},"ts":"1732218536342"} 2024-11-21T19:48:56,348 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=testBlockEvictionOnRegionMove1732218536256, state=ENABLING in hbase:meta 2024-11-21T19:48:56,349 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(204): Hosts are {7b96b36745c6=0} racks are {/default-rack=0} 2024-11-21T19:48:56,349 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-11-21T19:48:56,349 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-11-21T19:48:56,349 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-11-21T19:48:56,350 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-11-21T19:48:56,350 INFO [PEWorker-3 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-11-21T19:48:56,350 INFO [PEWorker-3 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-11-21T19:48:56,350 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-11-21T19:48:56,352 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1732218536256, region=7b850c849b1ef94018414fda0d571719, ASSIGN}] 2024-11-21T19:48:56,353 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1732218536256, region=7b850c849b1ef94018414fda0d571719, ASSIGN 2024-11-21T19:48:56,354 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1732218536256, region=7b850c849b1ef94018414fda0d571719, ASSIGN; state=OFFLINE, location=7b96b36745c6,43053,1732218534126; forceNewPlan=false, retain=false 2024-11-21T19:48:56,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39849 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-21T19:48:56,507 INFO [7b96b36745c6:39849 {}] balancer.BaseLoadBalancer(388): Reassigned 1 regions. 1 retained the pre-restart assignment. 2024-11-21T19:48:56,508 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=7b850c849b1ef94018414fda0d571719, regionState=OPENING, regionLocation=7b96b36745c6,43053,1732218534126 2024-11-21T19:48:56,511 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-4-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1732218536256, region=7b850c849b1ef94018414fda0d571719, ASSIGN because future has completed 2024-11-21T19:48:56,512 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 7b850c849b1ef94018414fda0d571719, server=7b96b36745c6,43053,1732218534126}] 2024-11-21T19:48:56,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39849 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-21T19:48:56,669 INFO [RS_OPEN_REGION-regionserver/7b96b36745c6:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(132): Open testBlockEvictionOnRegionMove1732218536256,,1732218536279.7b850c849b1ef94018414fda0d571719. 2024-11-21T19:48:56,669 DEBUG [RS_OPEN_REGION-regionserver/7b96b36745c6:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7752): Opening region: {ENCODED => 7b850c849b1ef94018414fda0d571719, NAME => 'testBlockEvictionOnRegionMove1732218536256,,1732218536279.7b850c849b1ef94018414fda0d571719.', STARTKEY => '', ENDKEY => ''} 2024-11-21T19:48:56,670 DEBUG [RS_OPEN_REGION-regionserver/7b96b36745c6:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testBlockEvictionOnRegionMove1732218536256 7b850c849b1ef94018414fda0d571719 2024-11-21T19:48:56,670 DEBUG [RS_OPEN_REGION-regionserver/7b96b36745c6:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(898): Instantiated testBlockEvictionOnRegionMove1732218536256,,1732218536279.7b850c849b1ef94018414fda0d571719.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-21T19:48:56,671 DEBUG [RS_OPEN_REGION-regionserver/7b96b36745c6:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7794): checking encryption for 7b850c849b1ef94018414fda0d571719 2024-11-21T19:48:56,671 DEBUG [RS_OPEN_REGION-regionserver/7b96b36745c6:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7797): checking classloading for 7b850c849b1ef94018414fda0d571719 2024-11-21T19:48:56,672 INFO [StoreOpener-7b850c849b1ef94018414fda0d571719-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family family of region 7b850c849b1ef94018414fda0d571719 2024-11-21T19:48:56,677 INFO [StoreOpener-7b850c849b1ef94018414fda0d571719-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 7b850c849b1ef94018414fda0d571719 columnFamilyName family 2024-11-21T19:48:56,677 DEBUG [StoreOpener-7b850c849b1ef94018414fda0d571719-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T19:48:56,678 INFO [StoreOpener-7b850c849b1ef94018414fda0d571719-1 {}] regionserver.HStore(327): Store=7b850c849b1ef94018414fda0d571719/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-21T19:48:56,678 DEBUG [RS_OPEN_REGION-regionserver/7b96b36745c6:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1038): replaying wal for 7b850c849b1ef94018414fda0d571719 2024-11-21T19:48:56,680 DEBUG [RS_OPEN_REGION-regionserver/7b96b36745c6:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0b774921-3780-c936-4fe1-1d1dcb4ce861/545083c6-d774-d2ca-7409-0fbc2c22cf1a/data/default/testBlockEvictionOnRegionMove1732218536256/7b850c849b1ef94018414fda0d571719 2024-11-21T19:48:56,681 DEBUG [RS_OPEN_REGION-regionserver/7b96b36745c6:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0b774921-3780-c936-4fe1-1d1dcb4ce861/545083c6-d774-d2ca-7409-0fbc2c22cf1a/data/default/testBlockEvictionOnRegionMove1732218536256/7b850c849b1ef94018414fda0d571719 2024-11-21T19:48:56,681 DEBUG [RS_OPEN_REGION-regionserver/7b96b36745c6:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1048): stopping wal replay for 7b850c849b1ef94018414fda0d571719 2024-11-21T19:48:56,681 DEBUG [RS_OPEN_REGION-regionserver/7b96b36745c6:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1060): Cleaning up temporary data for 7b850c849b1ef94018414fda0d571719 2024-11-21T19:48:56,682 DEBUG [RS_OPEN_REGION-regionserver/7b96b36745c6:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1093): writing seq id for 7b850c849b1ef94018414fda0d571719 2024-11-21T19:48:56,696 DEBUG [RS_OPEN_REGION-regionserver/7b96b36745c6:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0b774921-3780-c936-4fe1-1d1dcb4ce861/545083c6-d774-d2ca-7409-0fbc2c22cf1a/data/default/testBlockEvictionOnRegionMove1732218536256/7b850c849b1ef94018414fda0d571719/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-21T19:48:56,696 INFO [RS_OPEN_REGION-regionserver/7b96b36745c6:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1114): Opened 7b850c849b1ef94018414fda0d571719; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=64991366, jitterRate=-0.031553179025650024}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-21T19:48:56,697 DEBUG [RS_OPEN_REGION-regionserver/7b96b36745c6:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 7b850c849b1ef94018414fda0d571719 2024-11-21T19:48:56,698 DEBUG [RS_OPEN_REGION-regionserver/7b96b36745c6:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1006): Region open journal for 7b850c849b1ef94018414fda0d571719: Running coprocessor pre-open hook at 1732218536671Writing region info on filesystem at 1732218536671Initializing all the Stores at 1732218536671Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732218536671Cleaning up temporary data from old regions at 1732218536681 (+10 ms)Running coprocessor post-open hooks at 1732218536697 (+16 ms)Region opened successfully at 1732218536698 (+1 ms) 2024-11-21T19:48:56,700 INFO [RS_OPEN_REGION-regionserver/7b96b36745c6:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2236): Post open deploy tasks for testBlockEvictionOnRegionMove1732218536256,,1732218536279.7b850c849b1ef94018414fda0d571719., pid=6, masterSystemTime=1732218536665 2024-11-21T19:48:56,704 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=7b850c849b1ef94018414fda0d571719, regionState=OPEN, openSeqNum=2, regionLocation=7b96b36745c6,43053,1732218534126 2024-11-21T19:48:56,709 DEBUG [RS_OPEN_REGION-regionserver/7b96b36745c6:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2266): Finished post open deploy task for testBlockEvictionOnRegionMove1732218536256,,1732218536279.7b850c849b1ef94018414fda0d571719. 2024-11-21T19:48:56,709 INFO [RS_OPEN_REGION-regionserver/7b96b36745c6:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(153): Opened testBlockEvictionOnRegionMove1732218536256,,1732218536279.7b850c849b1ef94018414fda0d571719. 2024-11-21T19:48:56,710 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-4-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 7b850c849b1ef94018414fda0d571719, server=7b96b36745c6,43053,1732218534126 because future has completed 2024-11-21T19:48:56,714 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=6, resume processing ppid=5 2024-11-21T19:48:56,715 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure 7b850c849b1ef94018414fda0d571719, server=7b96b36745c6,43053,1732218534126 in 199 msec 2024-11-21T19:48:56,718 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=5, resume processing ppid=4 2024-11-21T19:48:56,718 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1732218536256, region=7b850c849b1ef94018414fda0d571719, ASSIGN in 364 msec 2024-11-21T19:48:56,719 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=testBlockEvictionOnRegionMove1732218536256 execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-21T19:48:56,720 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testBlockEvictionOnRegionMove1732218536256","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732218536719"}]},"ts":"1732218536719"} 2024-11-21T19:48:56,723 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=testBlockEvictionOnRegionMove1732218536256, state=ENABLED in hbase:meta 2024-11-21T19:48:56,724 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=testBlockEvictionOnRegionMove1732218536256 execute state=CREATE_TABLE_POST_OPERATION 2024-11-21T19:48:56,727 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=testBlockEvictionOnRegionMove1732218536256 in 437 msec 2024-11-21T19:48:56,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39849 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-21T19:48:56,927 INFO [RPCClient-NioEventLoopGroup-5-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:testBlockEvictionOnRegionMove1732218536256 completed 2024-11-21T19:48:56,927 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table testBlockEvictionOnRegionMove1732218536256 get assigned. Timeout = 60000ms 2024-11-21T19:48:56,930 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-21T19:48:56,936 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table testBlockEvictionOnRegionMove1732218536256 assigned to meta. Checking AM states. 2024-11-21T19:48:56,936 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-21T19:48:56,937 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table testBlockEvictionOnRegionMove1732218536256 assigned. 2024-11-21T19:48:56,946 DEBUG [RPCClient-NioEventLoopGroup-5-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testBlockEvictionOnRegionMove1732218536256', row='row1', locateType=CURRENT is [region=testBlockEvictionOnRegionMove1732218536256,,1732218536279.7b850c849b1ef94018414fda0d571719., hostname=7b96b36745c6,43053,1732218534126, seqNum=2] 2024-11-21T19:48:56,960 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 7b850c849b1ef94018414fda0d571719 1/1 column families, dataSize=78 B heapSize=480 B 2024-11-21T19:48:56,995 DEBUG [Time-limited test {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on efa94d47918a4e50af440503882e79e7 2024-11-21T19:48:57,045 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0b774921-3780-c936-4fe1-1d1dcb4ce861/545083c6-d774-d2ca-7409-0fbc2c22cf1a/data/default/testBlockEvictionOnRegionMove1732218536256/7b850c849b1ef94018414fda0d571719/.tmp/family/efa94d47918a4e50af440503882e79e7 is 43, key is row1/family:qf1/1/Put/seqid=0 2024-11-21T19:48:57,052 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=78 B at sequenceid=6 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0b774921-3780-c936-4fe1-1d1dcb4ce861/545083c6-d774-d2ca-7409-0fbc2c22cf1a/data/default/testBlockEvictionOnRegionMove1732218536256/7b850c849b1ef94018414fda0d571719/.tmp/family/efa94d47918a4e50af440503882e79e7 2024-11-21T19:48:57,089 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0b774921-3780-c936-4fe1-1d1dcb4ce861/545083c6-d774-d2ca-7409-0fbc2c22cf1a/data/default/testBlockEvictionOnRegionMove1732218536256/7b850c849b1ef94018414fda0d571719/.tmp/family/efa94d47918a4e50af440503882e79e7 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0b774921-3780-c936-4fe1-1d1dcb4ce861/545083c6-d774-d2ca-7409-0fbc2c22cf1a/data/default/testBlockEvictionOnRegionMove1732218536256/7b850c849b1ef94018414fda0d571719/family/efa94d47918a4e50af440503882e79e7 2024-11-21T19:48:57,091 DEBUG [Time-limited test {}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0b774921-3780-c936-4fe1-1d1dcb4ce861/545083c6-d774-d2ca-7409-0fbc2c22cf1a/data/default/testBlockEvictionOnRegionMove1732218536256/7b850c849b1ef94018414fda0d571719/family/efa94d47918a4e50af440503882e79e7, delay=984 ms 2024-11-21T19:48:57,095 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0b774921-3780-c936-4fe1-1d1dcb4ce861/545083c6-d774-d2ca-7409-0fbc2c22cf1a/data/default/testBlockEvictionOnRegionMove1732218536256/7b850c849b1ef94018414fda0d571719/family/efa94d47918a4e50af440503882e79e7, entries=2, sequenceid=6, filesize=4.9 K 2024-11-21T19:48:57,104 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~78 B/78, heapSize ~464 B/464, currentSize=0 B/0 for 7b850c849b1ef94018414fda0d571719 in 142ms, sequenceid=6, compaction requested=false 2024-11-21T19:48:57,105 DEBUG [Time-limited test {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'testBlockEvictionOnRegionMove1732218536256' 2024-11-21T19:48:57,107 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for 7b850c849b1ef94018414fda0d571719: 2024-11-21T19:48:58,077 DEBUG [hfile-prefetch-1732218537091 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key efa94d47918a4e50af440503882e79e7_0. 2024-11-21T19:48:58,082 DEBUG [hfile-prefetch-1732218537091 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key efa94d47918a4e50af440503882e79e7_117. 2024-11-21T19:48:58,083 DEBUG [hfile-prefetch-1732218537091 {}] bucket.BucketCache(2281): Notifying caching completed for file file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0b774921-3780-c936-4fe1-1d1dcb4ce861/545083c6-d774-d2ca-7409-0fbc2c22cf1a/data/default/testBlockEvictionOnRegionMove1732218536256/7b850c849b1ef94018414fda0d571719/family/efa94d47918a4e50af440503882e79e7, with total blocks 2, and data blocks 1 2024-11-21T19:48:58,083 DEBUG [hfile-prefetch-1732218537091 {}] bucket.BucketCache(2285): iterating over 0 entries in the backing map 2024-11-21T19:48:58,084 DEBUG [hfile-prefetch-1732218537091 {}] bucket.BucketCache(2308): Prefetch executor completed for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0b774921-3780-c936-4fe1-1d1dcb4ce861/545083c6-d774-d2ca-7409-0fbc2c22cf1a/data/default/testBlockEvictionOnRegionMove1732218536256/7b850c849b1ef94018414fda0d571719/family/efa94d47918a4e50af440503882e79e7, but only 0 data blocks were cached. Total data blocks for file: 1. Checking for blocks pending cache in cache writer queue. 2024-11-21T19:48:58,084 DEBUG [hfile-prefetch-1732218537091 {}] bucket.BucketCache(2318): There are still blocks pending caching for file file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0b774921-3780-c936-4fe1-1d1dcb4ce861/545083c6-d774-d2ca-7409-0fbc2c22cf1a/data/default/testBlockEvictionOnRegionMove1732218536256/7b850c849b1ef94018414fda0d571719/family/efa94d47918a4e50af440503882e79e7. Will sleep 100ms and try the verification again. 2024-11-21T19:48:58,087 DEBUG [bucket-cache-persister {}] bucket.BucketCachePersister(47): Cache is inconsistent, persisting to disk 2024-11-21T19:48:58,087 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1487): Thread bucket-cache-persister started persisting bucket cache to file 2024-11-21T19:48:58,087 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1494): Persist in new chunked persistence format. 2024-11-21T19:48:58,087 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1673): persistToFile: before persisting backing map size: 0, fullycachedFiles size: 0, chunkSize: 10000 2024-11-21T19:48:58,106 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1680): persistToFile: after persisting backing map size: 0, fullycachedFiles size: 0 2024-11-21T19:48:58,106 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1498): PersistToFile: after persisting backing map size: 0, fullycachedFiles size: 0, file name: bucket.persistence1732218538087 2024-11-21T19:48:58,106 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1509): Thread bucket-cache-persister finished persisting bucket cache to file, renaming 2024-11-21T19:48:58,119 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39849 {}] master.HMaster(2410): Client=jenkins//172.17.0.2 move hri=7b850c849b1ef94018414fda0d571719, source=7b96b36745c6,43053,1732218534126, destination=7b96b36745c6,33427,1732218534004, warming up region on 7b96b36745c6,33427,1732218534004 2024-11-21T19:48:58,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39849 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-21T19:48:58,123 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39849 {}] master.HMaster(2414): Client=jenkins//172.17.0.2 move hri=7b850c849b1ef94018414fda0d571719, source=7b96b36745c6,43053,1732218534126, destination=7b96b36745c6,33427,1732218534004, running balancer 2024-11-21T19:48:58,125 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48941, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-21T19:48:58,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39849 {}] procedure2.ProcedureExecutor(1139): Stored pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1732218536256, region=7b850c849b1ef94018414fda0d571719, REOPEN/MOVE 2024-11-21T19:48:58,125 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1732218536256, region=7b850c849b1ef94018414fda0d571719, REOPEN/MOVE 2024-11-21T19:48:58,127 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33427 {}] regionserver.RSRpcServices(2066): Warmup testBlockEvictionOnRegionMove1732218536256,,1732218536279.7b850c849b1ef94018414fda0d571719. 2024-11-21T19:48:58,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33427 {}] regionserver.HRegion(7855): Warmup {ENCODED => 7b850c849b1ef94018414fda0d571719, NAME => 'testBlockEvictionOnRegionMove1732218536256,,1732218536279.7b850c849b1ef94018414fda0d571719.', STARTKEY => '', ENDKEY => ''} 2024-11-21T19:48:58,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33427 {}] regionserver.HRegion(898): Instantiated testBlockEvictionOnRegionMove1732218536256,,1732218536279.7b850c849b1ef94018414fda0d571719.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-21T19:48:58,128 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=7b850c849b1ef94018414fda0d571719, regionState=CLOSING, regionLocation=7b96b36745c6,43053,1732218534126 2024-11-21T19:48:58,131 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-4-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1732218536256, region=7b850c849b1ef94018414fda0d571719, REOPEN/MOVE because future has completed 2024-11-21T19:48:58,133 INFO [StoreOpener-7b850c849b1ef94018414fda0d571719-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family family of region 7b850c849b1ef94018414fda0d571719 2024-11-21T19:48:58,134 INFO [StoreOpener-7b850c849b1ef94018414fda0d571719-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 7b850c849b1ef94018414fda0d571719 columnFamilyName family 2024-11-21T19:48:58,134 DEBUG [StoreOpener-7b850c849b1ef94018414fda0d571719-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T19:48:58,135 DEBUG [PEWorker-3 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: true 2024-11-21T19:48:58,136 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=8, ppid=7, state=RUNNABLE, hasLock=false; CloseRegionProcedure 7b850c849b1ef94018414fda0d571719, server=7b96b36745c6,43053,1732218534126}] 2024-11-21T19:48:58,141 DEBUG [StoreOpener-7b850c849b1ef94018414fda0d571719-1 {}] regionserver.StoreEngine(278): loaded file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0b774921-3780-c936-4fe1-1d1dcb4ce861/545083c6-d774-d2ca-7409-0fbc2c22cf1a/data/default/testBlockEvictionOnRegionMove1732218536256/7b850c849b1ef94018414fda0d571719/family/efa94d47918a4e50af440503882e79e7 2024-11-21T19:48:58,141 INFO [StoreOpener-7b850c849b1ef94018414fda0d571719-1 {}] regionserver.HStore(327): Store=7b850c849b1ef94018414fda0d571719/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-21T19:48:58,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33427 {}] regionserver.HRegion(1722): Closing 7b850c849b1ef94018414fda0d571719, disabling compactions & flushes 2024-11-21T19:48:58,142 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33427 {}] regionserver.HRegion(1755): Closing region testBlockEvictionOnRegionMove1732218536256,,1732218536279.7b850c849b1ef94018414fda0d571719. 2024-11-21T19:48:58,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33427 {}] regionserver.HRegion(1776): Time limited wait for close lock on testBlockEvictionOnRegionMove1732218536256,,1732218536279.7b850c849b1ef94018414fda0d571719. 2024-11-21T19:48:58,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33427 {}] regionserver.HRegion(1843): Acquired close lock on testBlockEvictionOnRegionMove1732218536256,,1732218536279.7b850c849b1ef94018414fda0d571719. after waiting 0 ms 2024-11-21T19:48:58,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33427 {}] regionserver.HRegion(1853): Updates disabled for region testBlockEvictionOnRegionMove1732218536256,,1732218536279.7b850c849b1ef94018414fda0d571719. 2024-11-21T19:48:58,147 DEBUG [StoreFileCloser-7b850c849b1ef94018414fda0d571719-family-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0b774921-3780-c936-4fe1-1d1dcb4ce861/545083c6-d774-d2ca-7409-0fbc2c22cf1a/data/default/testBlockEvictionOnRegionMove1732218536256/7b850c849b1ef94018414fda0d571719/family/efa94d47918a4e50af440503882e79e7 2024-11-21T19:48:58,148 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33427 {}] regionserver.HRegion(1973): Closed testBlockEvictionOnRegionMove1732218536256,,1732218536279.7b850c849b1ef94018414fda0d571719. 2024-11-21T19:48:58,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33427 {}] regionserver.HRegion(1676): Region close journal for 7b850c849b1ef94018414fda0d571719: Waiting for close lock at 1732218538142Disabling compacts and flushes for region at 1732218538142Disabling writes for close at 1732218538142Writing region close event to WAL at 1732218538148 (+6 ms)Closed at 1732218538148 2024-11-21T19:48:58,150 WARN [hfile-prefetch-1732218537091 {}] hfile.HFilePreadReader$1(158): Prefetch path=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0b774921-3780-c936-4fe1-1d1dcb4ce861/545083c6-d774-d2ca-7409-0fbc2c22cf1a/data/default/testBlockEvictionOnRegionMove1732218536256/7b850c849b1ef94018414fda0d571719/family/efa94d47918a4e50af440503882e79e7, offset=158, end=158 java.lang.RuntimeException: java.lang.InterruptedException: sleep interrupted at org.apache.hadoop.hbase.io.hfile.bucket.BucketCache.notifyFileCachingCompleted(BucketCache.java:2337) ~[classes/:?] at org.apache.hadoop.hbase.io.hfile.CombinedBlockCache.notifyFileCachingCompleted(CombinedBlockCache.java:469) ~[classes/:?] at org.apache.hadoop.hbase.io.hfile.HFilePreadReader$1.run(HFilePreadReader.java:148) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.lambda$tracedRunnable$2(TraceUtil.java:155) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:539) ~[?:?] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:304) ~[?:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCache.notifyFileCachingCompleted(BucketCache.java:2320) ~[classes/:?] ... 9 more 2024-11-21T19:48:58,151 DEBUG [hfile-prefetch-1732218537091 {}] hfile.PrefetchExecutor(121): Prefetch completed for efa94d47918a4e50af440503882e79e7 2024-11-21T19:48:58,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39849 {}] procedure.ProcedureSyncWait(219): waitFor pid=7 2024-11-21T19:48:58,293 INFO [RS_CLOSE_REGION-regionserver/7b96b36745c6:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] handler.UnassignRegionHandler(122): Close 7b850c849b1ef94018414fda0d571719 2024-11-21T19:48:58,293 DEBUG [RS_CLOSE_REGION-regionserver/7b96b36745c6:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: true 2024-11-21T19:48:58,294 DEBUG [RS_CLOSE_REGION-regionserver/7b96b36745c6:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1722): Closing 7b850c849b1ef94018414fda0d571719, disabling compactions & flushes 2024-11-21T19:48:58,294 INFO [RS_CLOSE_REGION-regionserver/7b96b36745c6:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1755): Closing region testBlockEvictionOnRegionMove1732218536256,,1732218536279.7b850c849b1ef94018414fda0d571719. 2024-11-21T19:48:58,294 DEBUG [RS_CLOSE_REGION-regionserver/7b96b36745c6:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1776): Time limited wait for close lock on testBlockEvictionOnRegionMove1732218536256,,1732218536279.7b850c849b1ef94018414fda0d571719. 2024-11-21T19:48:58,295 DEBUG [RS_CLOSE_REGION-regionserver/7b96b36745c6:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1843): Acquired close lock on testBlockEvictionOnRegionMove1732218536256,,1732218536279.7b850c849b1ef94018414fda0d571719. after waiting 0 ms 2024-11-21T19:48:58,295 DEBUG [RS_CLOSE_REGION-regionserver/7b96b36745c6:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1853): Updates disabled for region testBlockEvictionOnRegionMove1732218536256,,1732218536279.7b850c849b1ef94018414fda0d571719. 2024-11-21T19:48:58,297 DEBUG [StoreFileCloser-7b850c849b1ef94018414fda0d571719-family-1 {}] bucket.BucketCache(1863): found 1 blocks for file efa94d47918a4e50af440503882e79e7, starting offset: 0, end offset: 9223372036854775807 2024-11-21T19:48:58,298 DEBUG [StoreFileCloser-7b850c849b1ef94018414fda0d571719-family-1 {}] bucket.BucketCache(856): removed key efa94d47918a4e50af440503882e79e7_0 from back map with offset lock 2094080 in the evict process 2024-11-21T19:48:58,305 DEBUG [RS_CLOSE_REGION-regionserver/7b96b36745c6:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0b774921-3780-c936-4fe1-1d1dcb4ce861/545083c6-d774-d2ca-7409-0fbc2c22cf1a/data/default/testBlockEvictionOnRegionMove1732218536256/7b850c849b1ef94018414fda0d571719/recovered.edits/9.seqid, newMaxSeqId=9, maxSeqId=1 2024-11-21T19:48:58,306 INFO [RS_CLOSE_REGION-regionserver/7b96b36745c6:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1973): Closed testBlockEvictionOnRegionMove1732218536256,,1732218536279.7b850c849b1ef94018414fda0d571719. 2024-11-21T19:48:58,307 DEBUG [bucket-cache-persister {}] bucket.BucketCachePersister(47): Cache is inconsistent, persisting to disk 2024-11-21T19:48:58,307 DEBUG [RS_CLOSE_REGION-regionserver/7b96b36745c6:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1676): Region close journal for 7b850c849b1ef94018414fda0d571719: Waiting for close lock at 1732218538294Running coprocessor pre-close hooks at 1732218538294Disabling compacts and flushes for region at 1732218538294Disabling writes for close at 1732218538295 (+1 ms)Writing region close event to WAL at 1732218538299 (+4 ms)Running coprocessor post-close hooks at 1732218538305 (+6 ms)Closed at 1732218538306 (+1 ms) 2024-11-21T19:48:58,307 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1487): Thread bucket-cache-persister started persisting bucket cache to file 2024-11-21T19:48:58,307 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1494): Persist in new chunked persistence format. 2024-11-21T19:48:58,307 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1673): persistToFile: before persisting backing map size: 0, fullycachedFiles size: 0, chunkSize: 10000 2024-11-21T19:48:58,307 INFO [RS_CLOSE_REGION-regionserver/7b96b36745c6:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegionServer(3302): Adding 7b850c849b1ef94018414fda0d571719 move to 7b96b36745c6,33427,1732218534004 record at close sequenceid=6 2024-11-21T19:48:58,311 INFO [RS_CLOSE_REGION-regionserver/7b96b36745c6:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] handler.UnassignRegionHandler(157): Closed 7b850c849b1ef94018414fda0d571719 2024-11-21T19:48:58,311 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=7b850c849b1ef94018414fda0d571719, regionState=CLOSED 2024-11-21T19:48:58,312 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1680): persistToFile: after persisting backing map size: 0, fullycachedFiles size: 0 2024-11-21T19:48:58,312 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1498): PersistToFile: after persisting backing map size: 0, fullycachedFiles size: 0, file name: bucket.persistence1732218538307 2024-11-21T19:48:58,312 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1509): Thread bucket-cache-persister finished persisting bucket cache to file, renaming 2024-11-21T19:48:58,314 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-4-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=8, ppid=7, state=RUNNABLE, hasLock=false; CloseRegionProcedure 7b850c849b1ef94018414fda0d571719, server=7b96b36745c6,43053,1732218534126 because future has completed 2024-11-21T19:48:58,318 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=8, resume processing ppid=7 2024-11-21T19:48:58,318 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=8, ppid=7, state=SUCCESS, hasLock=false; CloseRegionProcedure 7b850c849b1ef94018414fda0d571719, server=7b96b36745c6,43053,1732218534126 in 179 msec 2024-11-21T19:48:58,318 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1732218536256, region=7b850c849b1ef94018414fda0d571719, REOPEN/MOVE; state=CLOSED, location=7b96b36745c6,33427,1732218534004; forceNewPlan=false, retain=false 2024-11-21T19:48:58,469 INFO [7b96b36745c6:39849 {}] balancer.BaseLoadBalancer(388): Reassigned 1 regions. 1 retained the pre-restart assignment. 2024-11-21T19:48:58,470 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=7b850c849b1ef94018414fda0d571719, regionState=OPENING, regionLocation=7b96b36745c6,33427,1732218534004 2024-11-21T19:48:58,473 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-4-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1732218536256, region=7b850c849b1ef94018414fda0d571719, REOPEN/MOVE because future has completed 2024-11-21T19:48:58,474 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=9, ppid=7, state=RUNNABLE, hasLock=false; OpenRegionProcedure 7b850c849b1ef94018414fda0d571719, server=7b96b36745c6,33427,1732218534004}] 2024-11-21T19:48:58,630 INFO [RS_OPEN_REGION-regionserver/7b96b36745c6:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] handler.AssignRegionHandler(132): Open testBlockEvictionOnRegionMove1732218536256,,1732218536279.7b850c849b1ef94018414fda0d571719. 2024-11-21T19:48:58,631 DEBUG [RS_OPEN_REGION-regionserver/7b96b36745c6:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(7752): Opening region: {ENCODED => 7b850c849b1ef94018414fda0d571719, NAME => 'testBlockEvictionOnRegionMove1732218536256,,1732218536279.7b850c849b1ef94018414fda0d571719.', STARTKEY => '', ENDKEY => ''} 2024-11-21T19:48:58,631 DEBUG [RS_OPEN_REGION-regionserver/7b96b36745c6:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testBlockEvictionOnRegionMove1732218536256 7b850c849b1ef94018414fda0d571719 2024-11-21T19:48:58,631 DEBUG [RS_OPEN_REGION-regionserver/7b96b36745c6:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(898): Instantiated testBlockEvictionOnRegionMove1732218536256,,1732218536279.7b850c849b1ef94018414fda0d571719.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-21T19:48:58,632 DEBUG [RS_OPEN_REGION-regionserver/7b96b36745c6:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(7794): checking encryption for 7b850c849b1ef94018414fda0d571719 2024-11-21T19:48:58,632 DEBUG [RS_OPEN_REGION-regionserver/7b96b36745c6:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(7797): checking classloading for 7b850c849b1ef94018414fda0d571719 2024-11-21T19:48:58,633 INFO [StoreOpener-7b850c849b1ef94018414fda0d571719-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family family of region 7b850c849b1ef94018414fda0d571719 2024-11-21T19:48:58,634 INFO [StoreOpener-7b850c849b1ef94018414fda0d571719-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 7b850c849b1ef94018414fda0d571719 columnFamilyName family 2024-11-21T19:48:58,634 DEBUG [StoreOpener-7b850c849b1ef94018414fda0d571719-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T19:48:58,636 DEBUG [StoreFileOpener-7b850c849b1ef94018414fda0d571719-family-1 {}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0b774921-3780-c936-4fe1-1d1dcb4ce861/545083c6-d774-d2ca-7409-0fbc2c22cf1a/data/default/testBlockEvictionOnRegionMove1732218536256/7b850c849b1ef94018414fda0d571719/family/efa94d47918a4e50af440503882e79e7, delay=938 ms 2024-11-21T19:48:58,638 DEBUG [StoreOpener-7b850c849b1ef94018414fda0d571719-1 {}] regionserver.StoreEngine(278): loaded file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0b774921-3780-c936-4fe1-1d1dcb4ce861/545083c6-d774-d2ca-7409-0fbc2c22cf1a/data/default/testBlockEvictionOnRegionMove1732218536256/7b850c849b1ef94018414fda0d571719/family/efa94d47918a4e50af440503882e79e7 2024-11-21T19:48:58,638 INFO [StoreOpener-7b850c849b1ef94018414fda0d571719-1 {}] regionserver.HStore(327): Store=7b850c849b1ef94018414fda0d571719/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-21T19:48:58,638 DEBUG [RS_OPEN_REGION-regionserver/7b96b36745c6:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1038): replaying wal for 7b850c849b1ef94018414fda0d571719 2024-11-21T19:48:58,640 DEBUG [RS_OPEN_REGION-regionserver/7b96b36745c6:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0b774921-3780-c936-4fe1-1d1dcb4ce861/545083c6-d774-d2ca-7409-0fbc2c22cf1a/data/default/testBlockEvictionOnRegionMove1732218536256/7b850c849b1ef94018414fda0d571719 2024-11-21T19:48:58,640 DEBUG [RS_OPEN_REGION-regionserver/7b96b36745c6:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0b774921-3780-c936-4fe1-1d1dcb4ce861/545083c6-d774-d2ca-7409-0fbc2c22cf1a/data/default/testBlockEvictionOnRegionMove1732218536256/7b850c849b1ef94018414fda0d571719 2024-11-21T19:48:58,640 DEBUG [RS_OPEN_REGION-regionserver/7b96b36745c6:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1048): stopping wal replay for 7b850c849b1ef94018414fda0d571719 2024-11-21T19:48:58,641 DEBUG [RS_OPEN_REGION-regionserver/7b96b36745c6:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1060): Cleaning up temporary data for 7b850c849b1ef94018414fda0d571719 2024-11-21T19:48:58,647 DEBUG [RS_OPEN_REGION-regionserver/7b96b36745c6:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1093): writing seq id for 7b850c849b1ef94018414fda0d571719 2024-11-21T19:48:58,648 INFO [RS_OPEN_REGION-regionserver/7b96b36745c6:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1114): Opened 7b850c849b1ef94018414fda0d571719; next sequenceid=10; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=73726177, jitterRate=0.09860564768314362}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-21T19:48:58,648 DEBUG [RS_OPEN_REGION-regionserver/7b96b36745c6:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 7b850c849b1ef94018414fda0d571719 2024-11-21T19:48:58,649 DEBUG [RS_OPEN_REGION-regionserver/7b96b36745c6:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1006): Region open journal for 7b850c849b1ef94018414fda0d571719: Running coprocessor pre-open hook at 1732218538632Writing region info on filesystem at 1732218538632Initializing all the Stores at 1732218538632Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732218538632Cleaning up temporary data from old regions at 1732218538641 (+9 ms)Running coprocessor post-open hooks at 1732218538648 (+7 ms)Region opened successfully at 1732218538649 (+1 ms) 2024-11-21T19:48:58,650 INFO [RS_OPEN_REGION-regionserver/7b96b36745c6:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegionServer(2236): Post open deploy tasks for testBlockEvictionOnRegionMove1732218536256,,1732218536279.7b850c849b1ef94018414fda0d571719., pid=9, masterSystemTime=1732218538626 2024-11-21T19:48:58,653 DEBUG [RS_OPEN_REGION-regionserver/7b96b36745c6:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegionServer(2266): Finished post open deploy task for testBlockEvictionOnRegionMove1732218536256,,1732218536279.7b850c849b1ef94018414fda0d571719. 2024-11-21T19:48:58,653 INFO [RS_OPEN_REGION-regionserver/7b96b36745c6:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] handler.AssignRegionHandler(153): Opened testBlockEvictionOnRegionMove1732218536256,,1732218536279.7b850c849b1ef94018414fda0d571719. 2024-11-21T19:48:58,653 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=7b850c849b1ef94018414fda0d571719, regionState=OPEN, openSeqNum=10, regionLocation=7b96b36745c6,33427,1732218534004 2024-11-21T19:48:58,656 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-4-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=9, ppid=7, state=RUNNABLE, hasLock=false; OpenRegionProcedure 7b850c849b1ef94018414fda0d571719, server=7b96b36745c6,33427,1732218534004 because future has completed 2024-11-21T19:48:58,660 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=9, resume processing ppid=7 2024-11-21T19:48:58,661 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=9, ppid=7, state=SUCCESS, hasLock=false; OpenRegionProcedure 7b850c849b1ef94018414fda0d571719, server=7b96b36745c6,33427,1732218534004 in 184 msec 2024-11-21T19:48:58,663 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=7, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1732218536256, region=7b850c849b1ef94018414fda0d571719, REOPEN/MOVE in 538 msec 2024-11-21T19:48:58,677 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster 2024-11-21T19:48:58,678 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-21T19:48:58,678 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79) at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611) at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.io.hfile.TestBlockEvictionOnRegionMovement.tearDown(TestBlockEvictionOnRegionMovement.java:174) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-21T19:48:58,684 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T19:48:58,685 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T19:48:58,685 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-21T19:48:58,685 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2024-11-21T19:48:58,685 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=587878705, stopped=false 2024-11-21T19:48:58,686 INFO [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=7b96b36745c6,39849,1732218533264 2024-11-21T19:48:58,689 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33427-0x10116d829200001, quorum=127.0.0.1:50077, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-21T19:48:58,690 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43053-0x10116d829200002, quorum=127.0.0.1:50077, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-21T19:48:58,690 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39849-0x10116d829200000, quorum=127.0.0.1:50077, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-21T19:48:58,690 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43053-0x10116d829200002, quorum=127.0.0.1:50077, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-21T19:48:58,690 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33427-0x10116d829200001, quorum=127.0.0.1:50077, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-21T19:48:58,690 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39849-0x10116d829200000, quorum=127.0.0.1:50077, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-21T19:48:58,690 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-21T19:48:58,691 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:33427-0x10116d829200001, quorum=127.0.0.1:50077, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-21T19:48:58,691 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:43053-0x10116d829200002, quorum=127.0.0.1:50077, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-21T19:48:58,691 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-21T19:48:58,692 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:39849-0x10116d829200000, quorum=127.0.0.1:50077, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-21T19:48:58,692 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277) at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265) at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416) at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.io.hfile.TestBlockEvictionOnRegionMovement.tearDown(TestBlockEvictionOnRegionMovement.java:174) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-21T19:48:58,692 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T19:48:58,693 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '7b96b36745c6,33427,1732218534004' ***** 2024-11-21T19:48:58,693 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-11-21T19:48:58,693 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '7b96b36745c6,43053,1732218534126' ***** 2024-11-21T19:48:58,694 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-11-21T19:48:58,694 INFO [RS:0;7b96b36745c6:33427 {}] bucket.BucketCache(1738): Disabling cache 2024-11-21T19:48:58,694 INFO [RS:1;7b96b36745c6:43053 {}] bucket.BucketCache(1738): Disabling cache 2024-11-21T19:48:58,694 INFO [RS:0;7b96b36745c6:33427 {}] bucket.BucketCache(1762): Shutdown bucket cache: IO persistent=true; path to write=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0b774921-3780-c936-4fe1-1d1dcb4ce861/bucket.persistence 2024-11-21T19:48:58,694 INFO [RS:1;7b96b36745c6:43053 {}] bucket.BucketCache(1762): Shutdown bucket cache: IO persistent=true; path to write=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0b774921-3780-c936-4fe1-1d1dcb4ce861/bucket.persistence 2024-11-21T19:48:58,694 INFO [Time-limited test-BucketCacheWriter-0 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-0 exiting, cacheEnabled=false 2024-11-21T19:48:58,694 INFO [Time-limited test-BucketCacheWriter-2 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-2 exiting, cacheEnabled=false 2024-11-21T19:48:58,694 INFO [Time-limited test-BucketCacheWriter-2 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-2 exiting, cacheEnabled=false 2024-11-21T19:48:58,694 INFO [Time-limited test-BucketCacheWriter-1 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-1 exiting, cacheEnabled=false 2024-11-21T19:48:58,694 INFO [Time-limited test-BucketCacheWriter-1 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-1 exiting, cacheEnabled=false 2024-11-21T19:48:58,695 INFO [Time-limited test-BucketCacheWriter-0 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-0 exiting, cacheEnabled=false 2024-11-21T19:48:58,695 INFO [RS:1;7b96b36745c6:43053 {}] bucket.BucketCache(1768): Shutting down cache persister thread. 2024-11-21T19:48:58,695 INFO [RS:0;7b96b36745c6:33427 {}] bucket.BucketCache(1768): Shutting down cache persister thread. 2024-11-21T19:48:58,695 WARN [bucket-cache-persister {}] bucket.BucketCachePersister(64): Interrupting BucketCachePersister thread. java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCachePersister.run(BucketCachePersister.java:45) ~[classes/:?] 2024-11-21T19:48:58,695 WARN [bucket-cache-persister {}] bucket.BucketCachePersister(64): Interrupting BucketCachePersister thread. java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCachePersister.run(BucketCachePersister.java:45) ~[classes/:?] 2024-11-21T19:48:58,705 DEBUG [RS:1;7b96b36745c6:43053 {}] bucket.BucketCache(1487): Thread RS:1;7b96b36745c6:43053 started persisting bucket cache to file 2024-11-21T19:48:58,705 DEBUG [RS:0;7b96b36745c6:33427 {}] bucket.BucketCache(1487): Thread RS:0;7b96b36745c6:33427 started persisting bucket cache to file 2024-11-21T19:48:58,705 DEBUG [RS:1;7b96b36745c6:43053 {}] bucket.BucketCache(1494): Persist in new chunked persistence format. 2024-11-21T19:48:58,705 DEBUG [RS:0;7b96b36745c6:33427 {}] bucket.BucketCache(1494): Persist in new chunked persistence format. 2024-11-21T19:48:58,705 DEBUG [RS:1;7b96b36745c6:43053 {}] bucket.BucketCache(1673): persistToFile: before persisting backing map size: 0, fullycachedFiles size: 0, chunkSize: 10000 2024-11-21T19:48:58,705 DEBUG [RS:0;7b96b36745c6:33427 {}] bucket.BucketCache(1673): persistToFile: before persisting backing map size: 0, fullycachedFiles size: 0, chunkSize: 10000 2024-11-21T19:48:58,710 WARN [Thread-95 {}] util.Shell$1(1036): Error reading the error stream java.io.IOException: Stream closed at java.io.BufferedInputStream.getBufIfOpen(BufferedInputStream.java:168) ~[?:?] at java.io.BufferedInputStream.read(BufferedInputStream.java:334) ~[?:?] at sun.nio.cs.StreamDecoder.readBytes(StreamDecoder.java:281) ~[?:?] at sun.nio.cs.StreamDecoder.implRead(StreamDecoder.java:324) ~[?:?] at sun.nio.cs.StreamDecoder.read(StreamDecoder.java:189) ~[?:?] at java.io.InputStreamReader.read(InputStreamReader.java:177) ~[?:?] at java.io.BufferedReader.fill(BufferedReader.java:162) ~[?:?] at java.io.BufferedReader.readLine(BufferedReader.java:329) ~[?:?] at java.io.BufferedReader.readLine(BufferedReader.java:396) ~[?:?] at org.apache.hadoop.util.Shell$1.run(Shell.java:1025) ~[hadoop-common-3.4.1.jar:?] 2024-11-21T19:48:58,710 ERROR [RS:0;7b96b36745c6:33427 {}] bucket.PersistentIOEngine(78): Calculating checksum failed, because of java.io.IOException: Stream closed at java.io.BufferedInputStream.getBufIfOpen(BufferedInputStream.java:168) ~[?:?] at java.io.BufferedInputStream.read(BufferedInputStream.java:334) ~[?:?] at sun.nio.cs.StreamDecoder.readBytes(StreamDecoder.java:281) ~[?:?] at sun.nio.cs.StreamDecoder.implRead(StreamDecoder.java:324) ~[?:?] at sun.nio.cs.StreamDecoder.read(StreamDecoder.java:189) ~[?:?] at java.io.InputStreamReader.read(InputStreamReader.java:177) ~[?:?] at java.io.BufferedReader.fill(BufferedReader.java:162) ~[?:?] at java.io.BufferedReader.read1(BufferedReader.java:213) ~[?:?] at java.io.BufferedReader.read(BufferedReader.java:287) ~[?:?] at org.apache.hadoop.util.Shell$ShellCommandExecutor.parseExecResult(Shell.java:1295) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.util.Shell.runCommand(Shell.java:1054) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.util.Shell.run(Shell.java:959) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.util.Shell$ShellCommandExecutor.execute(Shell.java:1282) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.io.hfile.bucket.PersistentIOEngine.getFileSize(PersistentIOEngine.java:94) ~[classes/:?] at org.apache.hadoop.hbase.io.hfile.bucket.PersistentIOEngine.calculateChecksum(PersistentIOEngine.java:71) ~[classes/:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketProtoUtils.toPB(BucketProtoUtils.java:61) ~[classes/:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketProtoUtils.serializeAsPB(BucketProtoUtils.java:75) ~[classes/:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCache.persistChunkedBackingMap(BucketCache.java:1678) ~[classes/:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCache.persistToFile(BucketCache.java:1496) ~[classes/:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCache.shutdown(BucketCache.java:1774) ~[classes/:?] at org.apache.hadoop.hbase.io.hfile.CombinedBlockCache.shutdown(CombinedBlockCache.java:169) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:927) ~[classes/:?] at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) ~[test-classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) ~[test-classes/:4.0.0-alpha-1-SNAPSHOT] at java.security.AccessController.doPrivileged(AccessController.java:399) ~[?:?] at javax.security.auth.Subject.doAs(Subject.java:376) ~[?:?] at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) ~[test-classes/:4.0.0-alpha-1-SNAPSHOT] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T19:48:58,710 DEBUG [RS:0;7b96b36745c6:33427 {}] bucket.BucketCache(1680): persistToFile: after persisting backing map size: 0, fullycachedFiles size: 0 2024-11-21T19:48:58,710 DEBUG [RS:0;7b96b36745c6:33427 {}] bucket.BucketCache(1498): PersistToFile: after persisting backing map size: 0, fullycachedFiles size: 0, file name: bucket.persistence1732218538705 2024-11-21T19:48:58,710 DEBUG [RS:0;7b96b36745c6:33427 {}] bucket.BucketCache(1509): Thread RS:0;7b96b36745c6:33427 finished persisting bucket cache to file, renaming 2024-11-21T19:48:58,711 INFO [RS:0;7b96b36745c6:33427 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-21T19:48:58,711 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-21T19:48:58,711 INFO [RS:0;7b96b36745c6:33427 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-21T19:48:58,711 INFO [RS:0;7b96b36745c6:33427 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-21T19:48:58,711 INFO [RS:0;7b96b36745c6:33427 {}] regionserver.HRegionServer(3091): Received CLOSE for 7b850c849b1ef94018414fda0d571719 2024-11-21T19:48:58,712 INFO [RS:0;7b96b36745c6:33427 {}] regionserver.HRegionServer(959): stopping server 7b96b36745c6,33427,1732218534004 2024-11-21T19:48:58,712 INFO [RS:0;7b96b36745c6:33427 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-21T19:48:58,712 INFO [RS:0;7b96b36745c6:33427 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;7b96b36745c6:33427. 2024-11-21T19:48:58,712 DEBUG [RS:0;7b96b36745c6:33427 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-21T19:48:58,712 DEBUG [RS:0;7b96b36745c6:33427 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T19:48:58,712 DEBUG [RS:1;7b96b36745c6:43053 {}] bucket.BucketCache(1680): persistToFile: after persisting backing map size: 0, fullycachedFiles size: 0 2024-11-21T19:48:58,712 INFO [RS_CLOSE_REGION-regionserver/7b96b36745c6:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1662): Closing region testBlockEvictionOnRegionMove1732218536256,,1732218536279.7b850c849b1ef94018414fda0d571719. during a graceful stop, and cache persistence is on, so setting evict on close to false. 2024-11-21T19:48:58,712 DEBUG [RS:1;7b96b36745c6:43053 {}] bucket.BucketCache(1498): PersistToFile: after persisting backing map size: 0, fullycachedFiles size: 0, file name: bucket.persistence1732218538705 2024-11-21T19:48:58,713 DEBUG [RS:1;7b96b36745c6:43053 {}] bucket.BucketCache(1509): Thread RS:1;7b96b36745c6:43053 finished persisting bucket cache to file, renaming 2024-11-21T19:48:58,713 INFO [RS:0;7b96b36745c6:33427 {}] regionserver.HRegionServer(1321): Waiting on 1 regions to close 2024-11-21T19:48:58,713 WARN [RS:1;7b96b36745c6:43053 {}] bucket.BucketCache(1512): Failed to commit cache persistent file. We might lose cached blocks if RS crashes/restarts before we successfully checkpoint again. 2024-11-21T19:48:58,713 DEBUG [RS:0;7b96b36745c6:33427 {}] regionserver.HRegionServer(1325): Online Regions={7b850c849b1ef94018414fda0d571719=testBlockEvictionOnRegionMove1732218536256,,1732218536279.7b850c849b1ef94018414fda0d571719.} 2024-11-21T19:48:58,713 DEBUG [RS_CLOSE_REGION-regionserver/7b96b36745c6:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 7b850c849b1ef94018414fda0d571719, disabling compactions & flushes 2024-11-21T19:48:58,713 INFO [RS:1;7b96b36745c6:43053 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-21T19:48:58,713 INFO [RS_CLOSE_REGION-regionserver/7b96b36745c6:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testBlockEvictionOnRegionMove1732218536256,,1732218536279.7b850c849b1ef94018414fda0d571719. 2024-11-21T19:48:58,713 DEBUG [RS_CLOSE_REGION-regionserver/7b96b36745c6:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testBlockEvictionOnRegionMove1732218536256,,1732218536279.7b850c849b1ef94018414fda0d571719. 2024-11-21T19:48:58,713 INFO [RS:1;7b96b36745c6:43053 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-21T19:48:58,713 DEBUG [RS:0;7b96b36745c6:33427 {}] regionserver.HRegionServer(1351): Waiting on 7b850c849b1ef94018414fda0d571719 2024-11-21T19:48:58,713 INFO [RS:1;7b96b36745c6:43053 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-21T19:48:58,713 DEBUG [RS_CLOSE_REGION-regionserver/7b96b36745c6:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testBlockEvictionOnRegionMove1732218536256,,1732218536279.7b850c849b1ef94018414fda0d571719. after waiting 0 ms 2024-11-21T19:48:58,713 INFO [RS:1;7b96b36745c6:43053 {}] regionserver.HRegionServer(959): stopping server 7b96b36745c6,43053,1732218534126 2024-11-21T19:48:58,713 DEBUG [RS_CLOSE_REGION-regionserver/7b96b36745c6:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testBlockEvictionOnRegionMove1732218536256,,1732218536279.7b850c849b1ef94018414fda0d571719. 2024-11-21T19:48:58,713 INFO [RS:1;7b96b36745c6:43053 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-21T19:48:58,713 INFO [RS:1;7b96b36745c6:43053 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:1;7b96b36745c6:43053. 2024-11-21T19:48:58,713 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-21T19:48:58,714 DEBUG [RS:1;7b96b36745c6:43053 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-21T19:48:58,714 DEBUG [RS:1;7b96b36745c6:43053 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T19:48:58,714 INFO [RS:1;7b96b36745c6:43053 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-21T19:48:58,714 INFO [RS:1;7b96b36745c6:43053 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-21T19:48:58,714 INFO [RS:1;7b96b36745c6:43053 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-21T19:48:58,714 DEBUG [StoreFileCloser-7b850c849b1ef94018414fda0d571719-family-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0b774921-3780-c936-4fe1-1d1dcb4ce861/545083c6-d774-d2ca-7409-0fbc2c22cf1a/data/default/testBlockEvictionOnRegionMove1732218536256/7b850c849b1ef94018414fda0d571719/family/efa94d47918a4e50af440503882e79e7 2024-11-21T19:48:58,714 INFO [RS:1;7b96b36745c6:43053 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-11-21T19:48:58,716 INFO [RS:1;7b96b36745c6:43053 {}] regionserver.HRegionServer(1321): Waiting on 1 regions to close 2024-11-21T19:48:58,716 DEBUG [RS:1;7b96b36745c6:43053 {}] regionserver.HRegionServer(1325): Online Regions={1588230740=hbase:meta,,1.1588230740} 2024-11-21T19:48:58,716 DEBUG [RS:1;7b96b36745c6:43053 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-21T19:48:58,716 INFO [RS_CLOSE_META-regionserver/7b96b36745c6:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1662): Closing region hbase:meta,,1.1588230740 during a graceful stop, and cache persistence is on, so setting evict on close to false. 2024-11-21T19:48:58,717 DEBUG [RS_CLOSE_META-regionserver/7b96b36745c6:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-21T19:48:58,717 INFO [RS_CLOSE_META-regionserver/7b96b36745c6:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-21T19:48:58,717 DEBUG [RS_CLOSE_META-regionserver/7b96b36745c6:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-21T19:48:58,717 DEBUG [RS_CLOSE_META-regionserver/7b96b36745c6:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-21T19:48:58,717 DEBUG [RS_CLOSE_META-regionserver/7b96b36745c6:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-21T19:48:58,717 INFO [RS_CLOSE_META-regionserver/7b96b36745c6:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=3.70 KB heapSize=6.72 KB 2024-11-21T19:48:58,720 DEBUG [RS_CLOSE_REGION-regionserver/7b96b36745c6:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0b774921-3780-c936-4fe1-1d1dcb4ce861/545083c6-d774-d2ca-7409-0fbc2c22cf1a/data/default/testBlockEvictionOnRegionMove1732218536256/7b850c849b1ef94018414fda0d571719/recovered.edits/12.seqid, newMaxSeqId=12, maxSeqId=9 2024-11-21T19:48:58,720 INFO [RS_CLOSE_REGION-regionserver/7b96b36745c6:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testBlockEvictionOnRegionMove1732218536256,,1732218536279.7b850c849b1ef94018414fda0d571719. 2024-11-21T19:48:58,720 DEBUG [RS_CLOSE_REGION-regionserver/7b96b36745c6:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 7b850c849b1ef94018414fda0d571719: Waiting for close lock at 1732218538712Running coprocessor pre-close hooks at 1732218538713 (+1 ms)Disabling compacts and flushes for region at 1732218538713Disabling writes for close at 1732218538713Writing region close event to WAL at 1732218538715 (+2 ms)Running coprocessor post-close hooks at 1732218538720 (+5 ms)Closed at 1732218538720 2024-11-21T19:48:58,721 DEBUG [RS_CLOSE_REGION-regionserver/7b96b36745c6:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testBlockEvictionOnRegionMove1732218536256,,1732218536279.7b850c849b1ef94018414fda0d571719. 2024-11-21T19:48:58,723 INFO [regionserver/7b96b36745c6:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-21T19:48:58,727 INFO [regionserver/7b96b36745c6:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-21T19:48:58,732 DEBUG [RS_CLOSE_META-regionserver/7b96b36745c6:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFile$WriterFactory(308): Unable to set drop behind on a37e713c7db142c68c68666e69c3ef94 2024-11-21T19:48:58,752 DEBUG [RS_CLOSE_META-regionserver/7b96b36745c6:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0b774921-3780-c936-4fe1-1d1dcb4ce861/545083c6-d774-d2ca-7409-0fbc2c22cf1a/data/hbase/meta/1588230740/.tmp/info/a37e713c7db142c68c68666e69c3ef94 is 205, key is testBlockEvictionOnRegionMove1732218536256,,1732218536279.7b850c849b1ef94018414fda0d571719./info:regioninfo/1732218538653/Put/seqid=0 2024-11-21T19:48:58,753 INFO [RS_CLOSE_META-regionserver/7b96b36745c6:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=3.48 KB at sequenceid=15 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0b774921-3780-c936-4fe1-1d1dcb4ce861/545083c6-d774-d2ca-7409-0fbc2c22cf1a/data/hbase/meta/1588230740/.tmp/info/a37e713c7db142c68c68666e69c3ef94 2024-11-21T19:48:58,766 DEBUG [RS_CLOSE_META-regionserver/7b96b36745c6:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFile$WriterFactory(308): Unable to set drop behind on c5ab31085e054dd0a89067309ba27326 2024-11-21T19:48:58,786 DEBUG [RS_CLOSE_META-regionserver/7b96b36745c6:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0b774921-3780-c936-4fe1-1d1dcb4ce861/545083c6-d774-d2ca-7409-0fbc2c22cf1a/data/hbase/meta/1588230740/.tmp/ns/c5ab31085e054dd0a89067309ba27326 is 43, key is default/ns:d/1732218535985/Put/seqid=0 2024-11-21T19:48:58,787 INFO [RS_CLOSE_META-regionserver/7b96b36745c6:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=74 B at sequenceid=15 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0b774921-3780-c936-4fe1-1d1dcb4ce861/545083c6-d774-d2ca-7409-0fbc2c22cf1a/data/hbase/meta/1588230740/.tmp/ns/c5ab31085e054dd0a89067309ba27326 2024-11-21T19:48:58,798 DEBUG [RS_CLOSE_META-regionserver/7b96b36745c6:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 671b5faeac9747769fe0ed0a70ed1c45 2024-11-21T19:48:58,814 DEBUG [RS_CLOSE_META-regionserver/7b96b36745c6:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0b774921-3780-c936-4fe1-1d1dcb4ce861/545083c6-d774-d2ca-7409-0fbc2c22cf1a/data/hbase/meta/1588230740/.tmp/table/671b5faeac9747769fe0ed0a70ed1c45 is 78, key is testBlockEvictionOnRegionMove1732218536256/table:state/1732218536719/Put/seqid=0 2024-11-21T19:48:58,814 INFO [RS_CLOSE_META-regionserver/7b96b36745c6:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=148 B at sequenceid=15 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0b774921-3780-c936-4fe1-1d1dcb4ce861/545083c6-d774-d2ca-7409-0fbc2c22cf1a/data/hbase/meta/1588230740/.tmp/table/671b5faeac9747769fe0ed0a70ed1c45 2024-11-21T19:48:58,817 DEBUG [RS_CLOSE_META-regionserver/7b96b36745c6:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0b774921-3780-c936-4fe1-1d1dcb4ce861/545083c6-d774-d2ca-7409-0fbc2c22cf1a/data/hbase/meta/1588230740/.tmp/info/a37e713c7db142c68c68666e69c3ef94 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0b774921-3780-c936-4fe1-1d1dcb4ce861/545083c6-d774-d2ca-7409-0fbc2c22cf1a/data/hbase/meta/1588230740/info/a37e713c7db142c68c68666e69c3ef94 2024-11-21T19:48:58,819 DEBUG [RS_CLOSE_META-regionserver/7b96b36745c6:0-0 {event_type=M_RS_CLOSE_META}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0b774921-3780-c936-4fe1-1d1dcb4ce861/545083c6-d774-d2ca-7409-0fbc2c22cf1a/data/hbase/meta/1588230740/info/a37e713c7db142c68c68666e69c3ef94, delay=934 ms 2024-11-21T19:48:58,821 INFO [RS_CLOSE_META-regionserver/7b96b36745c6:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0b774921-3780-c936-4fe1-1d1dcb4ce861/545083c6-d774-d2ca-7409-0fbc2c22cf1a/data/hbase/meta/1588230740/info/a37e713c7db142c68c68666e69c3ef94, entries=15, sequenceid=15, filesize=7.6 K 2024-11-21T19:48:58,821 DEBUG [RS_CLOSE_META-regionserver/7b96b36745c6:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0b774921-3780-c936-4fe1-1d1dcb4ce861/545083c6-d774-d2ca-7409-0fbc2c22cf1a/data/hbase/meta/1588230740/.tmp/ns/c5ab31085e054dd0a89067309ba27326 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0b774921-3780-c936-4fe1-1d1dcb4ce861/545083c6-d774-d2ca-7409-0fbc2c22cf1a/data/hbase/meta/1588230740/ns/c5ab31085e054dd0a89067309ba27326 2024-11-21T19:48:58,823 DEBUG [RS_CLOSE_META-regionserver/7b96b36745c6:0-0 {event_type=M_RS_CLOSE_META}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0b774921-3780-c936-4fe1-1d1dcb4ce861/545083c6-d774-d2ca-7409-0fbc2c22cf1a/data/hbase/meta/1588230740/ns/c5ab31085e054dd0a89067309ba27326, delay=980 ms 2024-11-21T19:48:58,825 INFO [RS_CLOSE_META-regionserver/7b96b36745c6:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0b774921-3780-c936-4fe1-1d1dcb4ce861/545083c6-d774-d2ca-7409-0fbc2c22cf1a/data/hbase/meta/1588230740/ns/c5ab31085e054dd0a89067309ba27326, entries=2, sequenceid=15, filesize=5.0 K 2024-11-21T19:48:58,825 DEBUG [RS_CLOSE_META-regionserver/7b96b36745c6:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0b774921-3780-c936-4fe1-1d1dcb4ce861/545083c6-d774-d2ca-7409-0fbc2c22cf1a/data/hbase/meta/1588230740/.tmp/table/671b5faeac9747769fe0ed0a70ed1c45 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0b774921-3780-c936-4fe1-1d1dcb4ce861/545083c6-d774-d2ca-7409-0fbc2c22cf1a/data/hbase/meta/1588230740/table/671b5faeac9747769fe0ed0a70ed1c45 2024-11-21T19:48:58,827 DEBUG [RS_CLOSE_META-regionserver/7b96b36745c6:0-0 {event_type=M_RS_CLOSE_META}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0b774921-3780-c936-4fe1-1d1dcb4ce861/545083c6-d774-d2ca-7409-0fbc2c22cf1a/data/hbase/meta/1588230740/table/671b5faeac9747769fe0ed0a70ed1c45, delay=985 ms 2024-11-21T19:48:58,829 INFO [RS_CLOSE_META-regionserver/7b96b36745c6:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0b774921-3780-c936-4fe1-1d1dcb4ce861/545083c6-d774-d2ca-7409-0fbc2c22cf1a/data/hbase/meta/1588230740/table/671b5faeac9747769fe0ed0a70ed1c45, entries=2, sequenceid=15, filesize=5.3 K 2024-11-21T19:48:58,831 INFO [RS_CLOSE_META-regionserver/7b96b36745c6:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~3.70 KB/3789, heapSize ~6.42 KB/6576, currentSize=0 B/0 for 1588230740 in 114ms, sequenceid=15, compaction requested=false 2024-11-21T19:48:58,831 DEBUG [RS_CLOSE_META-regionserver/7b96b36745c6:0-0 {event_type=M_RS_CLOSE_META}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta' 2024-11-21T19:48:58,841 DEBUG [StoreFileCloser-1588230740-info-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0b774921-3780-c936-4fe1-1d1dcb4ce861/545083c6-d774-d2ca-7409-0fbc2c22cf1a/data/hbase/meta/1588230740/info/a37e713c7db142c68c68666e69c3ef94 2024-11-21T19:48:58,842 DEBUG [StoreFileCloser-1588230740-ns-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0b774921-3780-c936-4fe1-1d1dcb4ce861/545083c6-d774-d2ca-7409-0fbc2c22cf1a/data/hbase/meta/1588230740/ns/c5ab31085e054dd0a89067309ba27326 2024-11-21T19:48:58,843 DEBUG [StoreFileCloser-1588230740-table-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0b774921-3780-c936-4fe1-1d1dcb4ce861/545083c6-d774-d2ca-7409-0fbc2c22cf1a/data/hbase/meta/1588230740/table/671b5faeac9747769fe0ed0a70ed1c45 2024-11-21T19:48:58,849 DEBUG [RS_CLOSE_META-regionserver/7b96b36745c6:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0b774921-3780-c936-4fe1-1d1dcb4ce861/545083c6-d774-d2ca-7409-0fbc2c22cf1a/data/hbase/meta/1588230740/recovered.edits/18.seqid, newMaxSeqId=18, maxSeqId=1 2024-11-21T19:48:58,850 DEBUG [RS_CLOSE_META-regionserver/7b96b36745c6:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-21T19:48:58,850 INFO [RS_CLOSE_META-regionserver/7b96b36745c6:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-21T19:48:58,850 DEBUG [RS_CLOSE_META-regionserver/7b96b36745c6:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1732218538716Running coprocessor pre-close hooks at 1732218538717 (+1 ms)Disabling compacts and flushes for region at 1732218538717Disabling writes for close at 1732218538717Obtaining lock to block concurrent updates at 1732218538717Preparing flush snapshotting stores in 1588230740 at 1732218538717Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=3789, getHeapSize=6816, getOffHeapSize=0, getCellsCount=27 at 1732218538718 (+1 ms)Flushing stores of hbase:meta,,1.1588230740 at 1732218538718Flushing 1588230740/info: creating writer at 1732218538719 (+1 ms)Flushing 1588230740/info: appending metadata at 1732218538750 (+31 ms)Flushing 1588230740/info: closing flushed file at 1732218538750Flushing 1588230740/ns: creating writer at 1732218538757 (+7 ms)Flushing 1588230740/ns: appending metadata at 1732218538785 (+28 ms)Flushing 1588230740/ns: closing flushed file at 1732218538785Flushing 1588230740/table: creating writer at 1732218538791 (+6 ms)Flushing 1588230740/table: appending metadata at 1732218538813 (+22 ms)Flushing 1588230740/table: closing flushed file at 1732218538813Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@239837b2: reopening flushed file at 1732218538817 (+4 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@4e1de789: reopening flushed file at 1732218538821 (+4 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@7e166d3d: reopening flushed file at 1732218538825 (+4 ms)Finished flush of dataSize ~3.70 KB/3789, heapSize ~6.42 KB/6576, currentSize=0 B/0 for 1588230740 in 114ms, sequenceid=15, compaction requested=false at 1732218538831 (+6 ms)Writing region close event to WAL at 1732218538844 (+13 ms)Running coprocessor post-close hooks at 1732218538850 (+6 ms)Closed at 1732218538850 2024-11-21T19:48:58,851 DEBUG [RS_CLOSE_META-regionserver/7b96b36745c6:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-11-21T19:48:58,913 INFO [RS:0;7b96b36745c6:33427 {}] regionserver.HRegionServer(976): stopping server 7b96b36745c6,33427,1732218534004; all regions closed. 2024-11-21T19:48:58,917 INFO [RS:1;7b96b36745c6:43053 {}] regionserver.HRegionServer(976): stopping server 7b96b36745c6,43053,1732218534126; all regions closed. 2024-11-21T19:48:58,918 DEBUG [RS:0;7b96b36745c6:33427 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0b774921-3780-c936-4fe1-1d1dcb4ce861/545083c6-d774-d2ca-7409-0fbc2c22cf1a/oldWALs 2024-11-21T19:48:58,918 INFO [RS:0;7b96b36745c6:33427 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 7b96b36745c6%2C33427%2C1732218534004:(num 1732218535401) 2024-11-21T19:48:58,919 DEBUG [RS:0;7b96b36745c6:33427 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T19:48:58,919 INFO [RS:0;7b96b36745c6:33427 {}] regionserver.LeaseManager(133): Closed leases 2024-11-21T19:48:58,919 INFO [RS:0;7b96b36745c6:33427 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-21T19:48:58,919 INFO [RS:0;7b96b36745c6:33427 {}] hbase.ChoreService(370): Chore service for: regionserver/7b96b36745c6:0 had [ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-11-21T19:48:58,919 INFO [RS:0;7b96b36745c6:33427 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-21T19:48:58,919 INFO [RS:0;7b96b36745c6:33427 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-21T19:48:58,919 INFO [regionserver/7b96b36745c6:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-21T19:48:58,919 INFO [RS:0;7b96b36745c6:33427 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-21T19:48:58,919 INFO [RS:0;7b96b36745c6:33427 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-21T19:48:58,920 INFO [RS:0;7b96b36745c6:33427 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:33427 2024-11-21T19:48:58,924 DEBUG [RS:1;7b96b36745c6:43053 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0b774921-3780-c936-4fe1-1d1dcb4ce861/545083c6-d774-d2ca-7409-0fbc2c22cf1a/oldWALs 2024-11-21T19:48:58,924 INFO [RS:1;7b96b36745c6:43053 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 7b96b36745c6%2C43053%2C1732218534126.meta:.meta(num 1732218535836) 2024-11-21T19:48:58,927 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39849-0x10116d829200000, quorum=127.0.0.1:50077, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-21T19:48:58,927 DEBUG [RS:1;7b96b36745c6:43053 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0b774921-3780-c936-4fe1-1d1dcb4ce861/545083c6-d774-d2ca-7409-0fbc2c22cf1a/oldWALs 2024-11-21T19:48:58,927 INFO [RS:1;7b96b36745c6:43053 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 7b96b36745c6%2C43053%2C1732218534126:(num 1732218535401) 2024-11-21T19:48:58,927 DEBUG [RS:1;7b96b36745c6:43053 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T19:48:58,927 INFO [RS:1;7b96b36745c6:43053 {}] regionserver.LeaseManager(133): Closed leases 2024-11-21T19:48:58,927 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33427-0x10116d829200001, quorum=127.0.0.1:50077, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/7b96b36745c6,33427,1732218534004 2024-11-21T19:48:58,927 INFO [RS:1;7b96b36745c6:43053 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-21T19:48:58,928 INFO [RS:1;7b96b36745c6:43053 {}] hbase.ChoreService(370): Chore service for: regionserver/7b96b36745c6:0 had [ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-11-21T19:48:58,928 INFO [RS:1;7b96b36745c6:43053 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-21T19:48:58,928 INFO [RS:1;7b96b36745c6:43053 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:43053 2024-11-21T19:48:58,929 INFO [RS:0;7b96b36745c6:33427 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-21T19:48:58,933 INFO [regionserver/7b96b36745c6:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-21T19:48:58,933 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [7b96b36745c6,33427,1732218534004] 2024-11-21T19:48:58,935 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43053-0x10116d829200002, quorum=127.0.0.1:50077, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/7b96b36745c6,43053,1732218534126 2024-11-21T19:48:58,935 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39849-0x10116d829200000, quorum=127.0.0.1:50077, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-21T19:48:58,935 INFO [RS:1;7b96b36745c6:43053 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-21T19:48:58,936 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/7b96b36745c6,33427,1732218534004 already deleted, retry=false 2024-11-21T19:48:58,937 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; 7b96b36745c6,33427,1732218534004 expired; onlineServers=1 2024-11-21T19:48:58,937 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [7b96b36745c6,43053,1732218534126] 2024-11-21T19:48:58,939 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/7b96b36745c6,43053,1732218534126 already deleted, retry=false 2024-11-21T19:48:58,939 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; 7b96b36745c6,43053,1732218534126 expired; onlineServers=0 2024-11-21T19:48:58,939 INFO [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master '7b96b36745c6,39849,1732218533264' ***** 2024-11-21T19:48:58,939 INFO [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0 2024-11-21T19:48:58,940 INFO [M:0;7b96b36745c6:39849 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-21T19:48:58,940 INFO [M:0;7b96b36745c6:39849 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-21T19:48:58,940 DEBUG [M:0;7b96b36745c6:39849 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2024-11-21T19:48:58,940 DEBUG [M:0;7b96b36745c6:39849 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2024-11-21T19:48:58,940 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2024-11-21T19:48:58,940 DEBUG [master/7b96b36745c6:0:becomeActiveMaster-HFileCleaner.large.0-1732218535084 {}] cleaner.HFileCleaner(306): Exit Thread[master/7b96b36745c6:0:becomeActiveMaster-HFileCleaner.large.0-1732218535084,5,FailOnTimeoutGroup] 2024-11-21T19:48:58,940 DEBUG [master/7b96b36745c6:0:becomeActiveMaster-HFileCleaner.small.0-1732218535085 {}] cleaner.HFileCleaner(306): Exit Thread[master/7b96b36745c6:0:becomeActiveMaster-HFileCleaner.small.0-1732218535085,5,FailOnTimeoutGroup] 2024-11-21T19:48:58,940 INFO [M:0;7b96b36745c6:39849 {}] hbase.ChoreService(370): Chore service for: master/7b96b36745c6:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown 2024-11-21T19:48:58,940 INFO [M:0;7b96b36745c6:39849 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-21T19:48:58,941 DEBUG [M:0;7b96b36745c6:39849 {}] master.HMaster(1795): Stopping service threads 2024-11-21T19:48:58,941 INFO [M:0;7b96b36745c6:39849 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2024-11-21T19:48:58,941 INFO [M:0;7b96b36745c6:39849 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-21T19:48:58,941 INFO [M:0;7b96b36745c6:39849 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-21T19:48:58,942 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2024-11-21T19:48:58,943 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39849-0x10116d829200000, quorum=127.0.0.1:50077, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-11-21T19:48:58,943 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39849-0x10116d829200000, quorum=127.0.0.1:50077, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-21T19:48:58,943 DEBUG [M:0;7b96b36745c6:39849 {}] zookeeper.ZKUtil(347): master:39849-0x10116d829200000, quorum=127.0.0.1:50077, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error) 2024-11-21T19:48:58,943 WARN [M:0;7b96b36745c6:39849 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null 2024-11-21T19:48:58,944 INFO [M:0;7b96b36745c6:39849 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0b774921-3780-c936-4fe1-1d1dcb4ce861/545083c6-d774-d2ca-7409-0fbc2c22cf1a/.lastflushedseqids 2024-11-21T19:48:58,954 INFO [M:0;7b96b36745c6:39849 {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-21T19:48:58,954 INFO [M:0;7b96b36745c6:39849 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2024-11-21T19:48:58,954 DEBUG [M:0;7b96b36745c6:39849 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-21T19:48:58,954 INFO [M:0;7b96b36745c6:39849 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-21T19:48:58,954 DEBUG [M:0;7b96b36745c6:39849 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-21T19:48:58,954 DEBUG [M:0;7b96b36745c6:39849 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-21T19:48:58,954 DEBUG [M:0;7b96b36745c6:39849 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-21T19:48:58,954 INFO [M:0;7b96b36745c6:39849 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=34.67 KB heapSize=43.02 KB 2024-11-21T19:48:58,970 DEBUG [M:0;7b96b36745c6:39849 {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on d612d8995ce244069f7d2957589b2193 2024-11-21T19:48:58,983 DEBUG [M:0;7b96b36745c6:39849 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0b774921-3780-c936-4fe1-1d1dcb4ce861/545083c6-d774-d2ca-7409-0fbc2c22cf1a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/d612d8995ce244069f7d2957589b2193 is 82, key is hbase:meta,,1/info:regioninfo/1732218535909/Put/seqid=0 2024-11-21T19:48:58,984 INFO [M:0;7b96b36745c6:39849 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=504 B at sequenceid=84 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0b774921-3780-c936-4fe1-1d1dcb4ce861/545083c6-d774-d2ca-7409-0fbc2c22cf1a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/d612d8995ce244069f7d2957589b2193 2024-11-21T19:48:58,994 DEBUG [M:0;7b96b36745c6:39849 {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 4f0c3b0cdf1649e59bc2184a1ec9ca7e 2024-11-21T19:48:59,011 DEBUG [M:0;7b96b36745c6:39849 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0b774921-3780-c936-4fe1-1d1dcb4ce861/545083c6-d774-d2ca-7409-0fbc2c22cf1a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/4f0c3b0cdf1649e59bc2184a1ec9ca7e is 778, key is \x00\x00\x00\x00\x00\x00\x00\x04/proc:d/1732218536726/Put/seqid=0 2024-11-21T19:48:59,012 INFO [M:0;7b96b36745c6:39849 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=34.05 KB at sequenceid=84 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0b774921-3780-c936-4fe1-1d1dcb4ce861/545083c6-d774-d2ca-7409-0fbc2c22cf1a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/4f0c3b0cdf1649e59bc2184a1ec9ca7e 2024-11-21T19:48:59,023 DEBUG [M:0;7b96b36745c6:39849 {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 0056a7dd23a545238069f6bf11dd765d 2024-11-21T19:48:59,037 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33427-0x10116d829200001, quorum=127.0.0.1:50077, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-21T19:48:59,037 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33427-0x10116d829200001, quorum=127.0.0.1:50077, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-21T19:48:59,037 INFO [RS:0;7b96b36745c6:33427 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-21T19:48:59,037 INFO [RS:1;7b96b36745c6:43053 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-21T19:48:59,038 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43053-0x10116d829200002, quorum=127.0.0.1:50077, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-21T19:48:59,038 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43053-0x10116d829200002, quorum=127.0.0.1:50077, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-21T19:48:59,038 INFO [RS:0;7b96b36745c6:33427 {}] regionserver.HRegionServer(1031): Exiting; stopping=7b96b36745c6,33427,1732218534004; zookeeper connection closed. 2024-11-21T19:48:59,038 INFO [RS:1;7b96b36745c6:43053 {}] regionserver.HRegionServer(1031): Exiting; stopping=7b96b36745c6,43053,1732218534126; zookeeper connection closed. 2024-11-21T19:48:59,038 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@49c14a17 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@49c14a17 2024-11-21T19:48:59,038 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@7b883e5f {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@7b883e5f 2024-11-21T19:48:59,039 INFO [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 2 regionserver(s) complete 2024-11-21T19:48:59,043 DEBUG [M:0;7b96b36745c6:39849 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0b774921-3780-c936-4fe1-1d1dcb4ce861/545083c6-d774-d2ca-7409-0fbc2c22cf1a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/0056a7dd23a545238069f6bf11dd765d is 69, key is 7b96b36745c6,33427,1732218534004/rs:state/1732218535110/Put/seqid=0 2024-11-21T19:48:59,044 INFO [M:0;7b96b36745c6:39849 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=130 B at sequenceid=84 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0b774921-3780-c936-4fe1-1d1dcb4ce861/545083c6-d774-d2ca-7409-0fbc2c22cf1a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/0056a7dd23a545238069f6bf11dd765d 2024-11-21T19:48:59,047 DEBUG [M:0;7b96b36745c6:39849 {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0b774921-3780-c936-4fe1-1d1dcb4ce861/545083c6-d774-d2ca-7409-0fbc2c22cf1a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/d612d8995ce244069f7d2957589b2193 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0b774921-3780-c936-4fe1-1d1dcb4ce861/545083c6-d774-d2ca-7409-0fbc2c22cf1a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/d612d8995ce244069f7d2957589b2193 2024-11-21T19:48:59,049 INFO [M:0;7b96b36745c6:39849 {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0b774921-3780-c936-4fe1-1d1dcb4ce861/545083c6-d774-d2ca-7409-0fbc2c22cf1a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/d612d8995ce244069f7d2957589b2193, entries=8, sequenceid=84, filesize=5.5 K 2024-11-21T19:48:59,050 DEBUG [M:0;7b96b36745c6:39849 {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0b774921-3780-c936-4fe1-1d1dcb4ce861/545083c6-d774-d2ca-7409-0fbc2c22cf1a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/4f0c3b0cdf1649e59bc2184a1ec9ca7e as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0b774921-3780-c936-4fe1-1d1dcb4ce861/545083c6-d774-d2ca-7409-0fbc2c22cf1a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/4f0c3b0cdf1649e59bc2184a1ec9ca7e 2024-11-21T19:48:59,053 INFO [M:0;7b96b36745c6:39849 {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0b774921-3780-c936-4fe1-1d1dcb4ce861/545083c6-d774-d2ca-7409-0fbc2c22cf1a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/4f0c3b0cdf1649e59bc2184a1ec9ca7e, entries=9, sequenceid=84, filesize=6.5 K 2024-11-21T19:48:59,053 DEBUG [M:0;7b96b36745c6:39849 {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0b774921-3780-c936-4fe1-1d1dcb4ce861/545083c6-d774-d2ca-7409-0fbc2c22cf1a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/0056a7dd23a545238069f6bf11dd765d as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0b774921-3780-c936-4fe1-1d1dcb4ce861/545083c6-d774-d2ca-7409-0fbc2c22cf1a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/0056a7dd23a545238069f6bf11dd765d 2024-11-21T19:48:59,055 INFO [M:0;7b96b36745c6:39849 {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0b774921-3780-c936-4fe1-1d1dcb4ce861/545083c6-d774-d2ca-7409-0fbc2c22cf1a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/0056a7dd23a545238069f6bf11dd765d, entries=2, sequenceid=84, filesize=5.1 K 2024-11-21T19:48:59,056 INFO [M:0;7b96b36745c6:39849 {}] regionserver.HRegion(3140): Finished flush of dataSize ~34.67 KB/35506, heapSize ~42.73 KB/43752, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 102ms, sequenceid=84, compaction requested=false 2024-11-21T19:48:59,076 INFO [M:0;7b96b36745c6:39849 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-21T19:48:59,076 DEBUG [M:0;7b96b36745c6:39849 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1732218538954Disabling compacts and flushes for region at 1732218538954Disabling writes for close at 1732218538954Obtaining lock to block concurrent updates at 1732218538955 (+1 ms)Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1732218538955Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=35506, getHeapSize=43992, getOffHeapSize=0, getCellsCount=99 at 1732218538955Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1732218538955Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1732218538955Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1732218538983 (+28 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1732218538983Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1732218538987 (+4 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1732218539011 (+24 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1732218539011Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1732218539015 (+4 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1732218539042 (+27 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1732218539042Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@6116e7de: reopening flushed file at 1732218539046 (+4 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@36a3618a: reopening flushed file at 1732218539050 (+4 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@133d60d8: reopening flushed file at 1732218539053 (+3 ms)Finished flush of dataSize ~34.67 KB/35506, heapSize ~42.73 KB/43752, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 102ms, sequenceid=84, compaction requested=false at 1732218539056 (+3 ms)Writing region close event to WAL at 1732218539076 (+20 ms)Closed at 1732218539076 2024-11-21T19:48:59,082 INFO [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-21T19:48:59,082 INFO [M:0;7b96b36745c6:39849 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down. 2024-11-21T19:48:59,082 INFO [M:0;7b96b36745c6:39849 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:39849 2024-11-21T19:48:59,083 INFO [M:0;7b96b36745c6:39849 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-21T19:48:59,186 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39849-0x10116d829200000, quorum=127.0.0.1:50077, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-21T19:48:59,186 INFO [M:0;7b96b36745c6:39849 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-21T19:48:59,187 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39849-0x10116d829200000, quorum=127.0.0.1:50077, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-21T19:48:59,195 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers 2024-11-21T19:48:59,225 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down 2024-11-21T19:48:59,225 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-21T19:48:59,237 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: io.hfile.TestBlockEvictionOnRegionMovement#testBlockEvictionOnRegionMove Thread=49 (was 12) Potentially hanging thread: HMaster-EventLoopGroup-1-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-3-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RpcClient-timer-pool-0 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.waitForNextTick(HashedWheelTimer.java:598) app//org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:494) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-4-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: regionserver/7b96b36745c6:0.procedureResultReporter java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.RemoteProcedureResultReporter.run(RemoteProcedureResultReporter.java:75) Potentially hanging thread: region-location-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1170) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: org.apache.hadoop.fs.FileSystem$Statistics$StatisticsDataReferenceCleaner java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:155) java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:176) app//org.apache.hadoop.fs.FileSystem$Statistics$StatisticsDataReferenceCleaner.run(FileSystem.java:4171) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-4 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-4-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: SnapshotHandlerChoreCleaner java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: hfile-prefetch-1732218538823 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1177) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: master/7b96b36745c6:0:becomeActiveMaster-MemStoreChunkPool Statistics java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ForkJoinPool.commonPool-worker-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.ForkJoinPool.awaitWork(ForkJoinPool.java:1724) java.base@17.0.11/java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1623) java.base@17.0.11/java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:165) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-3-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Idle-Rpc-Conn-Sweeper-pool-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HBase-Metrics2-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ForkJoinPool.commonPool-worker-3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.ForkJoinPool.awaitWork(ForkJoinPool.java:1724) java.base@17.0.11/java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1623) java.base@17.0.11/java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:165) Potentially hanging thread: hfile-prefetch-1732218537091 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: hfile-prefetch-1732218538636 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1177) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: master/7b96b36745c6:0:becomeActiveMaster-MemStoreChunkPool Statistics java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HMaster-EventLoopGroup-1-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HMaster-EventLoopGroup-1-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Time-limited test.named-queue-events-pool-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) app//com.lmax.disruptor.BlockingWaitStrategy.waitFor(BlockingWaitStrategy.java:47) app//com.lmax.disruptor.ProcessingSequenceBarrier.waitFor(ProcessingSequenceBarrier.java:56) app//com.lmax.disruptor.BatchEventProcessor.processEvents(BatchEventProcessor.java:159) app//com.lmax.disruptor.BatchEventProcessor.run(BatchEventProcessor.java:125) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: region-location-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1170) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-4-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Monitor thread for TaskMonitor java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hbase.monitoring.TaskMonitor$MonitorRunnable.run(TaskMonitor.java:325) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Async-Client-Retry-Timer-pool-0 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.waitForNextTick(HashedWheelTimer.java:598) app//org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:494) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: regionserver/7b96b36745c6:0.procedureResultReporter java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.RemoteProcedureResultReporter.run(RemoteProcedureResultReporter.java:75) Potentially hanging thread: weak-ref-cleaner-strictcontextstorage java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:155) java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:176) app//io.opentelemetry.context.StrictContextStorage$PendingScopes.run(StrictContextStorage.java:269) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: hfile-prefetch-1732218538819 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1177) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ForkJoinPool.commonPool-worker-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkUntil(LockSupport.java:410) java.base@17.0.11/java.util.concurrent.ForkJoinPool.awaitWork(ForkJoinPool.java:1726) java.base@17.0.11/java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1623) java.base@17.0.11/java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:165) Potentially hanging thread: SessionTracker java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.zookeeper.server.SessionTrackerImpl.run(SessionTrackerImpl.java:163) Potentially hanging thread: Timer for 'HBase' metrics system java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.util.TimerThread.mainLoop(Timer.java:563) java.base@17.0.11/java.util.TimerThread.run(Timer.java:516) - Thread LEAK? -, OpenFileDescriptor=346 (was 287) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=480 (was 504), ProcessCount=11 (was 11), AvailableMemoryMB=12268 (was 9746) - AvailableMemoryMB LEAK? - 2024-11-21T19:48:59,245 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: io.hfile.TestBlockEvictionOnRegionMovement#testBlockEvictionOnGracefulStop Thread=49, OpenFileDescriptor=346, MaxFileDescriptor=1048576, SystemLoadAverage=480, ProcessCount=11, AvailableMemoryMB=12268 2024-11-21T19:48:59,245 INFO [Time-limited test {}] hbase.HBaseTestingUtil(401): System.getProperty("hadoop.log.dir") already set to: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0b774921-3780-c936-4fe1-1d1dcb4ce861/hadoop-log-dir so I do NOT create it in target/test-data/11a8a75e-d82a-1368-d32b-668baa66865a 2024-11-21T19:48:59,245 INFO [Time-limited test {}] hbase.HBaseTestingUtil(401): System.getProperty("hadoop.tmp.dir") already set to: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0b774921-3780-c936-4fe1-1d1dcb4ce861/hadoop-tmp-dir so I do NOT create it in target/test-data/11a8a75e-d82a-1368-d32b-668baa66865a 2024-11-21T19:48:59,246 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-21T19:48:59,249 INFO [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/11a8a75e-d82a-1368-d32b-668baa66865a/cluster_5ce78a00-3eca-ba0b-7836-7db203864e8d, deleteOnExit=true 2024-11-21T19:48:59,254 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/11a8a75e-d82a-1368-d32b-668baa66865a/cluster_5ce78a00-3eca-ba0b-7836-7db203864e8d/zookeeper_0, clientPort=54341, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/11a8a75e-d82a-1368-d32b-668baa66865a/cluster_5ce78a00-3eca-ba0b-7836-7db203864e8d/zookeeper_0/version-2, dataDirSize=457 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/11a8a75e-d82a-1368-d32b-668baa66865a/cluster_5ce78a00-3eca-ba0b-7836-7db203864e8d/zookeeper_0/version-2, dataLogSize=457 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0 2024-11-21T19:48:59,258 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=54341 2024-11-21T19:48:59,286 INFO [Time-limited test {}] util.FSUtils(489): Created version file at /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0b774921-3780-c936-4fe1-1d1dcb4ce861/545083c6-d774-d2ca-7409-0fbc2c22cf1a with version=8 2024-11-21T19:48:59,286 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1141): The hbase.fs.tmp.dir is set to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0b774921-3780-c936-4fe1-1d1dcb4ce861/545083c6-d774-d2ca-7409-0fbc2c22cf1a/hbase-staging 2024-11-21T19:48:59,290 INFO [Time-limited test {}] client.ConnectionUtils(128): master/7b96b36745c6:0 server-side Connection retries=45 2024-11-21T19:48:59,290 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-21T19:48:59,291 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-21T19:48:59,291 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-21T19:48:59,291 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-21T19:48:59,291 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-21T19:48:59,291 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService 2024-11-21T19:48:59,291 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-21T19:48:59,292 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:44549 2024-11-21T19:48:59,294 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:44549 connecting to ZooKeeper ensemble=127.0.0.1:54341 2024-11-21T19:48:59,303 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:445490x0, quorum=127.0.0.1:54341, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-21T19:48:59,304 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:44549-0x10116d8419f0000 connected 2024-11-21T19:48:59,341 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-21T19:48:59,341 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-21T19:48:59,342 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:44549-0x10116d8419f0000, quorum=127.0.0.1:54341, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-21T19:48:59,342 INFO [Time-limited test {}] master.HMaster(525): hbase.rootdir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0b774921-3780-c936-4fe1-1d1dcb4ce861/545083c6-d774-d2ca-7409-0fbc2c22cf1a, hbase.cluster.distributed=false 2024-11-21T19:48:59,344 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:44549-0x10116d8419f0000, quorum=127.0.0.1:54341, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-21T19:48:59,345 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=44549 2024-11-21T19:48:59,349 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=44549 2024-11-21T19:48:59,349 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=44549 2024-11-21T19:48:59,353 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=44549 2024-11-21T19:48:59,353 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=44549 2024-11-21T19:48:59,370 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/7b96b36745c6:0 server-side Connection retries=45 2024-11-21T19:48:59,370 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-21T19:48:59,370 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-21T19:48:59,370 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-21T19:48:59,370 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-21T19:48:59,370 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-21T19:48:59,370 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-21T19:48:59,370 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-21T19:48:59,371 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:43071 2024-11-21T19:48:59,373 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:43071 connecting to ZooKeeper ensemble=127.0.0.1:54341 2024-11-21T19:48:59,373 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-21T19:48:59,373 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-21T19:48:59,377 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:430710x0, quorum=127.0.0.1:54341, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-21T19:48:59,377 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:430710x0, quorum=127.0.0.1:54341, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-21T19:48:59,377 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:43071-0x10116d8419f0001 connected 2024-11-21T19:48:59,378 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-21T19:48:59,381 INFO [Time-limited test {}] bucket.FileIOEngine(99): Allocating cache 400 MB, on the path:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/11a8a75e-d82a-1368-d32b-668baa66865a/bucket.cache 2024-11-21T19:48:59,381 INFO [Time-limited test {}] bucket.BucketCache(364): Instantiating BucketCache with acceptableFactor: 0.95, minFactor: 0.85, extraFreeFactor: 0.1, singleFactor: 0.25, multiFactor: 0.5, memoryFactor: 0.25, useStrongRef: false 2024-11-21T19:48:59,382 INFO [Time-limited test {}] bucket.BucketCache(478): Starting BucketCachePersisterThread 2024-11-21T19:48:59,382 INFO [Time-limited test {}] bucket.BucketCachePersister(38): BucketCachePersister started with interval: 100 2024-11-21T19:48:59,382 INFO [Thread-118 {}] bucket.BucketCache(1530): Started retrieving bucket cache from file 2024-11-21T19:48:59,382 WARN [Thread-118 {}] bucket.BucketCache(1533): Persistence file missing! It's ok if it's first run after enabling persistent cache. 2024-11-21T19:48:59,383 INFO [Thread-118 {}] bucket.BucketAllocator(343): Cache totalSize=418148352, buckets=199, bucket capacity=2101248=(4*525312)=(FEWEST_ITEMS_IN_BUCKET*(largest configured bucketcache size)) 2024-11-21T19:48:59,383 INFO [Thread-118 {}] bucket.BucketCache(413): Persistent bucket cache recovery from /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/11a8a75e-d82a-1368-d32b-668baa66865a/bucket.persistence is complete. 2024-11-21T19:48:59,383 INFO [Time-limited test {}] bucket.BucketCache(403): Started bucket cache; ioengine=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/11a8a75e-d82a-1368-d32b-668baa66865a/bucket.cache, capacity=400 MB, blockSize=64 KB, writerThreadNum=3, writerQLen=64, persistencePath=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/11a8a75e-d82a-1368-d32b-668baa66865a/bucket.persistence, bucketAllocator=org.apache.hadoop.hbase.io.hfile.bucket.BucketAllocator 2024-11-21T19:48:59,384 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-21T19:48:59,385 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:43071-0x10116d8419f0001, quorum=127.0.0.1:54341, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-21T19:48:59,387 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:43071-0x10116d8419f0001, quorum=127.0.0.1:54341, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-21T19:48:59,392 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=43071 2024-11-21T19:48:59,393 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=43071 2024-11-21T19:48:59,395 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=43071 2024-11-21T19:48:59,396 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=43071 2024-11-21T19:48:59,396 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=43071 2024-11-21T19:48:59,413 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/7b96b36745c6:0 server-side Connection retries=45 2024-11-21T19:48:59,413 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-21T19:48:59,413 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-21T19:48:59,413 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-21T19:48:59,413 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-21T19:48:59,413 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-21T19:48:59,413 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-21T19:48:59,413 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-21T19:48:59,414 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:45609 2024-11-21T19:48:59,416 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:45609 connecting to ZooKeeper ensemble=127.0.0.1:54341 2024-11-21T19:48:59,416 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-21T19:48:59,416 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-21T19:48:59,421 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:456090x0, quorum=127.0.0.1:54341, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-21T19:48:59,422 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:456090x0, quorum=127.0.0.1:54341, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-21T19:48:59,422 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:45609-0x10116d8419f0002 connected 2024-11-21T19:48:59,422 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-21T19:48:59,425 INFO [Time-limited test {}] bucket.FileIOEngine(99): Allocating cache 400 MB, on the path:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/11a8a75e-d82a-1368-d32b-668baa66865a/bucket.cache 2024-11-21T19:48:59,425 INFO [Time-limited test {}] bucket.BucketCache(364): Instantiating BucketCache with acceptableFactor: 0.95, minFactor: 0.85, extraFreeFactor: 0.1, singleFactor: 0.25, multiFactor: 0.5, memoryFactor: 0.25, useStrongRef: false 2024-11-21T19:48:59,426 INFO [Time-limited test {}] bucket.BucketCache(478): Starting BucketCachePersisterThread 2024-11-21T19:48:59,426 INFO [Time-limited test {}] bucket.BucketCachePersister(38): BucketCachePersister started with interval: 100 2024-11-21T19:48:59,429 INFO [Thread-119 {}] bucket.BucketCache(1530): Started retrieving bucket cache from file 2024-11-21T19:48:59,429 INFO [Time-limited test {}] bucket.BucketCache(403): Started bucket cache; ioengine=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/11a8a75e-d82a-1368-d32b-668baa66865a/bucket.cache, capacity=400 MB, blockSize=64 KB, writerThreadNum=3, writerQLen=64, persistencePath=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/11a8a75e-d82a-1368-d32b-668baa66865a/bucket.persistence, bucketAllocator=org.apache.hadoop.hbase.io.hfile.bucket.BucketAllocator 2024-11-21T19:48:59,429 WARN [Thread-119 {}] bucket.BucketCache(1533): Persistence file missing! It's ok if it's first run after enabling persistent cache. 2024-11-21T19:48:59,430 INFO [Thread-119 {}] bucket.BucketAllocator(343): Cache totalSize=418148352, buckets=199, bucket capacity=2101248=(4*525312)=(FEWEST_ITEMS_IN_BUCKET*(largest configured bucketcache size)) 2024-11-21T19:48:59,430 INFO [Thread-119 {}] bucket.BucketCache(413): Persistent bucket cache recovery from /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/11a8a75e-d82a-1368-d32b-668baa66865a/bucket.persistence is complete. 2024-11-21T19:48:59,430 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-21T19:48:59,431 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:45609-0x10116d8419f0002, quorum=127.0.0.1:54341, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-21T19:48:59,433 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:45609-0x10116d8419f0002, quorum=127.0.0.1:54341, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-21T19:48:59,437 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=45609 2024-11-21T19:48:59,439 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=45609 2024-11-21T19:48:59,441 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=45609 2024-11-21T19:48:59,446 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=45609 2024-11-21T19:48:59,448 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=45609 2024-11-21T19:48:59,463 DEBUG [M:0;7b96b36745c6:44549 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;7b96b36745c6:44549 2024-11-21T19:48:59,464 INFO [master/7b96b36745c6:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/7b96b36745c6,44549,1732218539289 2024-11-21T19:48:59,466 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45609-0x10116d8419f0002, quorum=127.0.0.1:54341, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-21T19:48:59,467 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43071-0x10116d8419f0001, quorum=127.0.0.1:54341, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-21T19:48:59,467 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44549-0x10116d8419f0000, quorum=127.0.0.1:54341, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-21T19:48:59,468 DEBUG [master/7b96b36745c6:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:44549-0x10116d8419f0000, quorum=127.0.0.1:54341, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/7b96b36745c6,44549,1732218539289 2024-11-21T19:48:59,470 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43071-0x10116d8419f0001, quorum=127.0.0.1:54341, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-21T19:48:59,471 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43071-0x10116d8419f0001, quorum=127.0.0.1:54341, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-21T19:48:59,471 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45609-0x10116d8419f0002, quorum=127.0.0.1:54341, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-21T19:48:59,471 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45609-0x10116d8419f0002, quorum=127.0.0.1:54341, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-21T19:48:59,471 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44549-0x10116d8419f0000, quorum=127.0.0.1:54341, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-21T19:48:59,472 DEBUG [master/7b96b36745c6:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:44549-0x10116d8419f0000, quorum=127.0.0.1:54341, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-11-21T19:48:59,473 INFO [master/7b96b36745c6:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/7b96b36745c6,44549,1732218539289 from backup master directory 2024-11-21T19:48:59,474 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45609-0x10116d8419f0002, quorum=127.0.0.1:54341, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-21T19:48:59,474 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43071-0x10116d8419f0001, quorum=127.0.0.1:54341, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-21T19:48:59,475 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44549-0x10116d8419f0000, quorum=127.0.0.1:54341, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/7b96b36745c6,44549,1732218539289 2024-11-21T19:48:59,475 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44549-0x10116d8419f0000, quorum=127.0.0.1:54341, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-21T19:48:59,475 WARN [master/7b96b36745c6:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-21T19:48:59,475 INFO [master/7b96b36745c6:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=7b96b36745c6,44549,1732218539289 2024-11-21T19:48:59,477 DEBUG [master/7b96b36745c6:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0b774921-3780-c936-4fe1-1d1dcb4ce861/545083c6-d774-d2ca-7409-0fbc2c22cf1a/hbase.id] with ID: f26c0bbc-ffc3-4204-8c91-2c4d735cdc03 2024-11-21T19:48:59,477 DEBUG [master/7b96b36745c6:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0b774921-3780-c936-4fe1-1d1dcb4ce861/545083c6-d774-d2ca-7409-0fbc2c22cf1a/.tmp/hbase.id 2024-11-21T19:48:59,484 DEBUG [master/7b96b36745c6:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0b774921-3780-c936-4fe1-1d1dcb4ce861/545083c6-d774-d2ca-7409-0fbc2c22cf1a/.tmp/hbase.id]:[file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0b774921-3780-c936-4fe1-1d1dcb4ce861/545083c6-d774-d2ca-7409-0fbc2c22cf1a/hbase.id] 2024-11-21T19:48:59,520 DEBUG [master/7b96b36745c6:0:becomeActiveMaster {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-21T19:48:59,521 INFO [master/7b96b36745c6:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem. 2024-11-21T19:48:59,522 INFO [master/7b96b36745c6:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 1ms. 2024-11-21T19:48:59,525 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44549-0x10116d8419f0000, quorum=127.0.0.1:54341, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-21T19:48:59,525 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45609-0x10116d8419f0002, quorum=127.0.0.1:54341, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-21T19:48:59,525 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43071-0x10116d8419f0001, quorum=127.0.0.1:54341, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-21T19:48:59,530 INFO [master/7b96b36745c6:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-21T19:48:59,532 INFO [master/7b96b36745c6:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-11-21T19:48:59,532 INFO [master/7b96b36745c6:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-21T19:48:59,552 INFO [master/7b96b36745c6:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0b774921-3780-c936-4fe1-1d1dcb4ce861/545083c6-d774-d2ca-7409-0fbc2c22cf1a/MasterData/data/master/store 2024-11-21T19:48:59,558 DEBUG [master/7b96b36745c6:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-21T19:48:59,558 DEBUG [master/7b96b36745c6:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-21T19:48:59,558 INFO [master/7b96b36745c6:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-21T19:48:59,559 DEBUG [master/7b96b36745c6:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-21T19:48:59,559 DEBUG [master/7b96b36745c6:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-21T19:48:59,559 DEBUG [master/7b96b36745c6:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-21T19:48:59,559 INFO [master/7b96b36745c6:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-21T19:48:59,559 DEBUG [master/7b96b36745c6:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1732218539558Disabling compacts and flushes for region at 1732218539558Disabling writes for close at 1732218539559 (+1 ms)Writing region close event to WAL at 1732218539559Closed at 1732218539559 2024-11-21T19:48:59,559 WARN [master/7b96b36745c6:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0b774921-3780-c936-4fe1-1d1dcb4ce861/545083c6-d774-d2ca-7409-0fbc2c22cf1a/MasterData/data/master/store/.initializing 2024-11-21T19:48:59,559 DEBUG [master/7b96b36745c6:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0b774921-3780-c936-4fe1-1d1dcb4ce861/545083c6-d774-d2ca-7409-0fbc2c22cf1a/MasterData/WALs/7b96b36745c6,44549,1732218539289 2024-11-21T19:48:59,564 INFO [master/7b96b36745c6:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-21T19:48:59,568 INFO [master/7b96b36745c6:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=7b96b36745c6%2C44549%2C1732218539289, suffix=, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0b774921-3780-c936-4fe1-1d1dcb4ce861/545083c6-d774-d2ca-7409-0fbc2c22cf1a/MasterData/WALs/7b96b36745c6,44549,1732218539289, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0b774921-3780-c936-4fe1-1d1dcb4ce861/545083c6-d774-d2ca-7409-0fbc2c22cf1a/MasterData/oldWALs, maxLogs=10 2024-11-21T19:48:59,585 INFO [master/7b96b36745c6:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0b774921-3780-c936-4fe1-1d1dcb4ce861/545083c6-d774-d2ca-7409-0fbc2c22cf1a/MasterData/WALs/7b96b36745c6,44549,1732218539289/7b96b36745c6%2C44549%2C1732218539289.1732218539568 2024-11-21T19:48:59,586 DEBUG [master/7b96b36745c6:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-11-21T19:48:59,586 DEBUG [master/7b96b36745c6:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-11-21T19:48:59,586 DEBUG [master/7b96b36745c6:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-21T19:48:59,586 DEBUG [master/7b96b36745c6:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-11-21T19:48:59,586 DEBUG [master/7b96b36745c6:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-11-21T19:48:59,587 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-11-21T19:48:59,591 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-11-21T19:48:59,591 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T19:48:59,591 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-21T19:48:59,592 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-11-21T19:48:59,595 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-11-21T19:48:59,595 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T19:48:59,595 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-21T19:48:59,595 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-11-21T19:48:59,599 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-11-21T19:48:59,599 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T19:48:59,599 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-21T19:48:59,599 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-11-21T19:48:59,603 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-11-21T19:48:59,603 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T19:48:59,603 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-21T19:48:59,603 DEBUG [master/7b96b36745c6:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682 2024-11-21T19:48:59,604 DEBUG [master/7b96b36745c6:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0b774921-3780-c936-4fe1-1d1dcb4ce861/545083c6-d774-d2ca-7409-0fbc2c22cf1a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-21T19:48:59,604 DEBUG [master/7b96b36745c6:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0b774921-3780-c936-4fe1-1d1dcb4ce861/545083c6-d774-d2ca-7409-0fbc2c22cf1a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-21T19:48:59,604 DEBUG [master/7b96b36745c6:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682 2024-11-21T19:48:59,604 DEBUG [master/7b96b36745c6:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682 2024-11-21T19:48:59,604 DEBUG [master/7b96b36745c6:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-21T19:48:59,605 DEBUG [master/7b96b36745c6:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-11-21T19:48:59,612 DEBUG [master/7b96b36745c6:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0b774921-3780-c936-4fe1-1d1dcb4ce861/545083c6-d774-d2ca-7409-0fbc2c22cf1a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-21T19:48:59,612 INFO [master/7b96b36745c6:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=58898526, jitterRate=-0.12234356999397278}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-21T19:48:59,613 DEBUG [master/7b96b36745c6:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682: Writing region info on filesystem at 1732218539586Initializing all the Stores at 1732218539587 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732218539587Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732218539587Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732218539587Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732218539587Cleaning up temporary data from old regions at 1732218539604 (+17 ms)Region opened successfully at 1732218539613 (+9 ms) 2024-11-21T19:48:59,615 INFO [master/7b96b36745c6:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-11-21T19:48:59,621 DEBUG [master/7b96b36745c6:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@f261a7f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=7b96b36745c6/172.17.0.2:0 2024-11-21T19:48:59,622 INFO [master/7b96b36745c6:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating... 2024-11-21T19:48:59,622 INFO [master/7b96b36745c6:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5 2024-11-21T19:48:59,622 INFO [master/7b96b36745c6:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50 2024-11-21T19:48:59,622 INFO [master/7b96b36745c6:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-21T19:48:59,623 INFO [master/7b96b36745c6:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-21T19:48:59,623 INFO [master/7b96b36745c6:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 0 msec 2024-11-21T19:48:59,623 INFO [master/7b96b36745c6:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-11-21T19:48:59,626 INFO [master/7b96b36745c6:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2024-11-21T19:48:59,627 DEBUG [master/7b96b36745c6:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:44549-0x10116d8419f0000, quorum=127.0.0.1:54341, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error) 2024-11-21T19:48:59,629 DEBUG [master/7b96b36745c6:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false 2024-11-21T19:48:59,629 INFO [master/7b96b36745c6:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2024-11-21T19:48:59,630 DEBUG [master/7b96b36745c6:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:44549-0x10116d8419f0000, quorum=127.0.0.1:54341, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error) 2024-11-21T19:48:59,631 DEBUG [master/7b96b36745c6:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false 2024-11-21T19:48:59,631 INFO [master/7b96b36745c6:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2024-11-21T19:48:59,633 DEBUG [master/7b96b36745c6:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:44549-0x10116d8419f0000, quorum=127.0.0.1:54341, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error) 2024-11-21T19:48:59,634 DEBUG [master/7b96b36745c6:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false 2024-11-21T19:48:59,635 DEBUG [master/7b96b36745c6:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:44549-0x10116d8419f0000, quorum=127.0.0.1:54341, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error) 2024-11-21T19:48:59,636 DEBUG [master/7b96b36745c6:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false 2024-11-21T19:48:59,640 DEBUG [master/7b96b36745c6:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:44549-0x10116d8419f0000, quorum=127.0.0.1:54341, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error) 2024-11-21T19:48:59,642 DEBUG [master/7b96b36745c6:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false 2024-11-21T19:48:59,644 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45609-0x10116d8419f0002, quorum=127.0.0.1:54341, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-21T19:48:59,644 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44549-0x10116d8419f0000, quorum=127.0.0.1:54341, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-21T19:48:59,644 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43071-0x10116d8419f0001, quorum=127.0.0.1:54341, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-21T19:48:59,644 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45609-0x10116d8419f0002, quorum=127.0.0.1:54341, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-21T19:48:59,644 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43071-0x10116d8419f0001, quorum=127.0.0.1:54341, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-21T19:48:59,644 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44549-0x10116d8419f0000, quorum=127.0.0.1:54341, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-21T19:48:59,649 INFO [master/7b96b36745c6:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=7b96b36745c6,44549,1732218539289, sessionid=0x10116d8419f0000, setting cluster-up flag (Was=false) 2024-11-21T19:48:59,652 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43071-0x10116d8419f0001, quorum=127.0.0.1:54341, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-21T19:48:59,653 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45609-0x10116d8419f0002, quorum=127.0.0.1:54341, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-21T19:48:59,653 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44549-0x10116d8419f0000, quorum=127.0.0.1:54341, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-21T19:48:59,659 DEBUG [master/7b96b36745c6:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort 2024-11-21T19:48:59,661 DEBUG [master/7b96b36745c6:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=7b96b36745c6,44549,1732218539289 2024-11-21T19:48:59,663 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44549-0x10116d8419f0000, quorum=127.0.0.1:54341, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-21T19:48:59,663 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45609-0x10116d8419f0002, quorum=127.0.0.1:54341, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-21T19:48:59,663 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43071-0x10116d8419f0001, quorum=127.0.0.1:54341, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-21T19:48:59,669 DEBUG [master/7b96b36745c6:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort 2024-11-21T19:48:59,670 DEBUG [master/7b96b36745c6:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=7b96b36745c6,44549,1732218539289 2024-11-21T19:48:59,671 INFO [master/7b96b36745c6:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0b774921-3780-c936-4fe1-1d1dcb4ce861/545083c6-d774-d2ca-7409-0fbc2c22cf1a/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again 2024-11-21T19:48:59,672 DEBUG [master/7b96b36745c6:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-21T19:48:59,673 INFO [master/7b96b36745c6:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2 2024-11-21T19:48:59,673 INFO [master/7b96b36745c6:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2024-11-21T19:48:59,673 DEBUG [master/7b96b36745c6:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: 7b96b36745c6,44549,1732218539289 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2024-11-21T19:48:59,675 DEBUG [master/7b96b36745c6:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/7b96b36745c6:0, corePoolSize=5, maxPoolSize=5 2024-11-21T19:48:59,675 DEBUG [master/7b96b36745c6:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/7b96b36745c6:0, corePoolSize=5, maxPoolSize=5 2024-11-21T19:48:59,675 DEBUG [master/7b96b36745c6:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/7b96b36745c6:0, corePoolSize=5, maxPoolSize=5 2024-11-21T19:48:59,675 DEBUG [master/7b96b36745c6:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/7b96b36745c6:0, corePoolSize=5, maxPoolSize=5 2024-11-21T19:48:59,675 DEBUG [master/7b96b36745c6:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/7b96b36745c6:0, corePoolSize=10, maxPoolSize=10 2024-11-21T19:48:59,675 DEBUG [master/7b96b36745c6:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/7b96b36745c6:0, corePoolSize=1, maxPoolSize=1 2024-11-21T19:48:59,675 DEBUG [master/7b96b36745c6:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/7b96b36745c6:0, corePoolSize=2, maxPoolSize=2 2024-11-21T19:48:59,675 DEBUG [master/7b96b36745c6:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/7b96b36745c6:0, corePoolSize=1, maxPoolSize=1 2024-11-21T19:48:59,682 INFO [master/7b96b36745c6:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732218569682 2024-11-21T19:48:59,682 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-21T19:48:59,682 INFO [master/7b96b36745c6:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2024-11-21T19:48:59,682 INFO [master/7b96b36745c6:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2024-11-21T19:48:59,682 INFO [master/7b96b36745c6:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2024-11-21T19:48:59,682 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region 2024-11-21T19:48:59,682 INFO [master/7b96b36745c6:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2024-11-21T19:48:59,682 INFO [master/7b96b36745c6:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2024-11-21T19:48:59,682 INFO [master/7b96b36745c6:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2024-11-21T19:48:59,683 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T19:48:59,683 INFO [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-11-21T19:48:59,683 INFO [master/7b96b36745c6:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-21T19:48:59,684 INFO [master/7b96b36745c6:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2024-11-21T19:48:59,685 INFO [master/7b96b36745c6:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2024-11-21T19:48:59,685 INFO [master/7b96b36745c6:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2024-11-21T19:48:59,688 INFO [master/7b96b36745c6:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2024-11-21T19:48:59,689 INFO [master/7b96b36745c6:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2024-11-21T19:48:59,693 DEBUG [master/7b96b36745c6:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/7b96b36745c6:0:becomeActiveMaster-HFileCleaner.large.0-1732218539689,5,FailOnTimeoutGroup] 2024-11-21T19:48:59,697 DEBUG [master/7b96b36745c6:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/7b96b36745c6:0:becomeActiveMaster-HFileCleaner.small.0-1732218539693,5,FailOnTimeoutGroup] 2024-11-21T19:48:59,698 INFO [master/7b96b36745c6:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-21T19:48:59,698 INFO [master/7b96b36745c6:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2024-11-21T19:48:59,698 INFO [master/7b96b36745c6:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2024-11-21T19:48:59,698 INFO [master/7b96b36745c6:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2024-11-21T19:48:59,704 INFO [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0b774921-3780-c936-4fe1-1d1dcb4ce861/545083c6-d774-d2ca-7409-0fbc2c22cf1a/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321 2024-11-21T19:48:59,705 INFO [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0b774921-3780-c936-4fe1-1d1dcb4ce861/545083c6-d774-d2ca-7409-0fbc2c22cf1a 2024-11-21T19:48:59,718 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-21T19:48:59,719 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family info of region 1588230740 2024-11-21T19:48:59,726 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-21T19:48:59,726 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T19:48:59,726 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-21T19:48:59,727 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family ns of region 1588230740 2024-11-21T19:48:59,730 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-21T19:48:59,730 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T19:48:59,730 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-21T19:48:59,730 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family rep_barrier of region 1588230740 2024-11-21T19:48:59,734 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-21T19:48:59,734 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T19:48:59,735 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-21T19:48:59,735 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family table of region 1588230740 2024-11-21T19:48:59,739 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-21T19:48:59,739 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T19:48:59,739 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-21T19:48:59,739 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-21T19:48:59,740 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0b774921-3780-c936-4fe1-1d1dcb4ce861/545083c6-d774-d2ca-7409-0fbc2c22cf1a/data/hbase/meta/1588230740 2024-11-21T19:48:59,740 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0b774921-3780-c936-4fe1-1d1dcb4ce861/545083c6-d774-d2ca-7409-0fbc2c22cf1a/data/hbase/meta/1588230740 2024-11-21T19:48:59,740 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-21T19:48:59,740 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-21T19:48:59,741 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-21T19:48:59,741 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-21T19:48:59,748 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0b774921-3780-c936-4fe1-1d1dcb4ce861/545083c6-d774-d2ca-7409-0fbc2c22cf1a/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-21T19:48:59,749 INFO [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=67044066, jitterRate=-9.655654430389404E-4}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-21T19:48:59,750 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740: Writing region info on filesystem at 1732218539718Initializing all the Stores at 1732218539718Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732218539718Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732218539718Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732218539719 (+1 ms)Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732218539719Cleaning up temporary data from old regions at 1732218539740 (+21 ms)Region opened successfully at 1732218539750 (+10 ms) 2024-11-21T19:48:59,750 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-21T19:48:59,750 INFO [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-21T19:48:59,750 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-21T19:48:59,750 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-21T19:48:59,750 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-21T19:48:59,751 INFO [RS:1;7b96b36745c6:45609 {}] regionserver.HRegionServer(746): ClusterId : f26c0bbc-ffc3-4204-8c91-2c4d735cdc03 2024-11-21T19:48:59,751 INFO [RS:0;7b96b36745c6:43071 {}] regionserver.HRegionServer(746): ClusterId : f26c0bbc-ffc3-4204-8c91-2c4d735cdc03 2024-11-21T19:48:59,751 DEBUG [RS:0;7b96b36745c6:43071 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-21T19:48:59,751 DEBUG [RS:1;7b96b36745c6:45609 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-21T19:48:59,751 INFO [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-21T19:48:59,751 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1732218539750Disabling compacts and flushes for region at 1732218539750Disabling writes for close at 1732218539750Writing region close event to WAL at 1732218539751 (+1 ms)Closed at 1732218539751 2024-11-21T19:48:59,752 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-21T19:48:59,752 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta 2024-11-21T19:48:59,752 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-11-21T19:48:59,753 DEBUG [RS:0;7b96b36745c6:43071 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-21T19:48:59,753 DEBUG [RS:1;7b96b36745c6:45609 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-21T19:48:59,754 DEBUG [RS:0;7b96b36745c6:43071 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-21T19:48:59,754 DEBUG [RS:1;7b96b36745c6:45609 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-21T19:48:59,754 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-21T19:48:59,755 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2024-11-21T19:48:59,756 DEBUG [RS:0;7b96b36745c6:43071 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-21T19:48:59,756 DEBUG [RS:1;7b96b36745c6:45609 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-21T19:48:59,757 DEBUG [RS:0;7b96b36745c6:43071 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@63a9b4ed, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=7b96b36745c6/172.17.0.2:0 2024-11-21T19:48:59,757 DEBUG [RS:1;7b96b36745c6:45609 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@680782a1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=7b96b36745c6/172.17.0.2:0 2024-11-21T19:48:59,776 DEBUG [RS:1;7b96b36745c6:45609 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:1;7b96b36745c6:45609 2024-11-21T19:48:59,776 INFO [RS:1;7b96b36745c6:45609 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-21T19:48:59,776 INFO [RS:1;7b96b36745c6:45609 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-21T19:48:59,776 DEBUG [RS:0;7b96b36745c6:43071 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;7b96b36745c6:43071 2024-11-21T19:48:59,776 DEBUG [RS:1;7b96b36745c6:45609 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-21T19:48:59,776 INFO [RS:0;7b96b36745c6:43071 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-21T19:48:59,776 INFO [RS:0;7b96b36745c6:43071 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-21T19:48:59,776 DEBUG [RS:0;7b96b36745c6:43071 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-21T19:48:59,777 INFO [RS:1;7b96b36745c6:45609 {}] regionserver.HRegionServer(2659): reportForDuty to master=7b96b36745c6,44549,1732218539289 with port=45609, startcode=1732218539412 2024-11-21T19:48:59,777 INFO [RS:0;7b96b36745c6:43071 {}] regionserver.HRegionServer(2659): reportForDuty to master=7b96b36745c6,44549,1732218539289 with port=43071, startcode=1732218539369 2024-11-21T19:48:59,777 DEBUG [RS:0;7b96b36745c6:43071 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-21T19:48:59,777 DEBUG [RS:1;7b96b36745c6:45609 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-21T19:48:59,781 INFO [HMaster-EventLoopGroup-6-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40619, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=RegionServerStatusService 2024-11-21T19:48:59,781 INFO [HMaster-EventLoopGroup-6-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47887, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=RegionServerStatusService 2024-11-21T19:48:59,781 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=44549 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 7b96b36745c6,45609,1732218539412 2024-11-21T19:48:59,782 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=44549 {}] master.ServerManager(517): Registering regionserver=7b96b36745c6,45609,1732218539412 2024-11-21T19:48:59,783 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=44549 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 7b96b36745c6,43071,1732218539369 2024-11-21T19:48:59,783 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=44549 {}] master.ServerManager(517): Registering regionserver=7b96b36745c6,43071,1732218539369 2024-11-21T19:48:59,783 DEBUG [RS:1;7b96b36745c6:45609 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0b774921-3780-c936-4fe1-1d1dcb4ce861/545083c6-d774-d2ca-7409-0fbc2c22cf1a 2024-11-21T19:48:59,783 DEBUG [RS:1;7b96b36745c6:45609 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=file:/ 2024-11-21T19:48:59,783 DEBUG [RS:1;7b96b36745c6:45609 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-21T19:48:59,785 DEBUG [RS:0;7b96b36745c6:43071 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0b774921-3780-c936-4fe1-1d1dcb4ce861/545083c6-d774-d2ca-7409-0fbc2c22cf1a 2024-11-21T19:48:59,785 DEBUG [RS:0;7b96b36745c6:43071 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=file:/ 2024-11-21T19:48:59,785 DEBUG [RS:0;7b96b36745c6:43071 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-21T19:48:59,785 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44549-0x10116d8419f0000, quorum=127.0.0.1:54341, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-21T19:48:59,789 DEBUG [RS:1;7b96b36745c6:45609 {}] zookeeper.ZKUtil(111): regionserver:45609-0x10116d8419f0002, quorum=127.0.0.1:54341, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/7b96b36745c6,45609,1732218539412 2024-11-21T19:48:59,789 WARN [RS:1;7b96b36745c6:45609 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-21T19:48:59,790 INFO [RS:1;7b96b36745c6:45609 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-21T19:48:59,790 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [7b96b36745c6,45609,1732218539412] 2024-11-21T19:48:59,790 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [7b96b36745c6,43071,1732218539369] 2024-11-21T19:48:59,790 DEBUG [RS:1;7b96b36745c6:45609 {}] regionserver.HRegionServer(1793): logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0b774921-3780-c936-4fe1-1d1dcb4ce861/545083c6-d774-d2ca-7409-0fbc2c22cf1a/WALs/7b96b36745c6,45609,1732218539412 2024-11-21T19:48:59,790 DEBUG [RS:0;7b96b36745c6:43071 {}] zookeeper.ZKUtil(111): regionserver:43071-0x10116d8419f0001, quorum=127.0.0.1:54341, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/7b96b36745c6,43071,1732218539369 2024-11-21T19:48:59,790 WARN [RS:0;7b96b36745c6:43071 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-21T19:48:59,790 INFO [RS:0;7b96b36745c6:43071 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-21T19:48:59,790 DEBUG [RS:0;7b96b36745c6:43071 {}] regionserver.HRegionServer(1793): logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0b774921-3780-c936-4fe1-1d1dcb4ce861/545083c6-d774-d2ca-7409-0fbc2c22cf1a/WALs/7b96b36745c6,43071,1732218539369 2024-11-21T19:48:59,797 INFO [RS:0;7b96b36745c6:43071 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-21T19:48:59,797 INFO [RS:1;7b96b36745c6:45609 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-21T19:48:59,799 INFO [RS:0;7b96b36745c6:43071 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-21T19:48:59,801 INFO [RS:0;7b96b36745c6:43071 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-21T19:48:59,801 INFO [RS:0;7b96b36745c6:43071 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-21T19:48:59,805 INFO [RS:1;7b96b36745c6:45609 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-21T19:48:59,805 INFO [RS:0;7b96b36745c6:43071 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-21T19:48:59,806 INFO [RS:0;7b96b36745c6:43071 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-21T19:48:59,806 INFO [RS:0;7b96b36745c6:43071 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-21T19:48:59,806 DEBUG [RS:0;7b96b36745c6:43071 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/7b96b36745c6:0, corePoolSize=1, maxPoolSize=1 2024-11-21T19:48:59,806 DEBUG [RS:0;7b96b36745c6:43071 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/7b96b36745c6:0, corePoolSize=1, maxPoolSize=1 2024-11-21T19:48:59,806 DEBUG [RS:0;7b96b36745c6:43071 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/7b96b36745c6:0, corePoolSize=1, maxPoolSize=1 2024-11-21T19:48:59,806 DEBUG [RS:0;7b96b36745c6:43071 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/7b96b36745c6:0, corePoolSize=1, maxPoolSize=1 2024-11-21T19:48:59,806 DEBUG [RS:0;7b96b36745c6:43071 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/7b96b36745c6:0, corePoolSize=1, maxPoolSize=1 2024-11-21T19:48:59,806 DEBUG [RS:0;7b96b36745c6:43071 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/7b96b36745c6:0, corePoolSize=2, maxPoolSize=2 2024-11-21T19:48:59,807 DEBUG [RS:0;7b96b36745c6:43071 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/7b96b36745c6:0, corePoolSize=1, maxPoolSize=1 2024-11-21T19:48:59,807 DEBUG [RS:0;7b96b36745c6:43071 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/7b96b36745c6:0, corePoolSize=1, maxPoolSize=1 2024-11-21T19:48:59,807 DEBUG [RS:0;7b96b36745c6:43071 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/7b96b36745c6:0, corePoolSize=1, maxPoolSize=1 2024-11-21T19:48:59,807 DEBUG [RS:0;7b96b36745c6:43071 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/7b96b36745c6:0, corePoolSize=1, maxPoolSize=1 2024-11-21T19:48:59,807 DEBUG [RS:0;7b96b36745c6:43071 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/7b96b36745c6:0, corePoolSize=1, maxPoolSize=1 2024-11-21T19:48:59,807 DEBUG [RS:0;7b96b36745c6:43071 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/7b96b36745c6:0, corePoolSize=1, maxPoolSize=1 2024-11-21T19:48:59,807 DEBUG [RS:0;7b96b36745c6:43071 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/7b96b36745c6:0, corePoolSize=3, maxPoolSize=3 2024-11-21T19:48:59,807 DEBUG [RS:0;7b96b36745c6:43071 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/7b96b36745c6:0, corePoolSize=3, maxPoolSize=3 2024-11-21T19:48:59,808 INFO [RS:1;7b96b36745c6:45609 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-21T19:48:59,808 INFO [RS:1;7b96b36745c6:45609 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-21T19:48:59,809 INFO [RS:1;7b96b36745c6:45609 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-21T19:48:59,810 INFO [RS:1;7b96b36745c6:45609 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-21T19:48:59,810 INFO [RS:1;7b96b36745c6:45609 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-21T19:48:59,810 DEBUG [RS:1;7b96b36745c6:45609 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/7b96b36745c6:0, corePoolSize=1, maxPoolSize=1 2024-11-21T19:48:59,810 DEBUG [RS:1;7b96b36745c6:45609 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/7b96b36745c6:0, corePoolSize=1, maxPoolSize=1 2024-11-21T19:48:59,811 DEBUG [RS:1;7b96b36745c6:45609 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/7b96b36745c6:0, corePoolSize=1, maxPoolSize=1 2024-11-21T19:48:59,811 DEBUG [RS:1;7b96b36745c6:45609 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/7b96b36745c6:0, corePoolSize=1, maxPoolSize=1 2024-11-21T19:48:59,811 DEBUG [RS:1;7b96b36745c6:45609 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/7b96b36745c6:0, corePoolSize=1, maxPoolSize=1 2024-11-21T19:48:59,811 DEBUG [RS:1;7b96b36745c6:45609 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/7b96b36745c6:0, corePoolSize=2, maxPoolSize=2 2024-11-21T19:48:59,811 DEBUG [RS:1;7b96b36745c6:45609 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/7b96b36745c6:0, corePoolSize=1, maxPoolSize=1 2024-11-21T19:48:59,811 DEBUG [RS:1;7b96b36745c6:45609 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/7b96b36745c6:0, corePoolSize=1, maxPoolSize=1 2024-11-21T19:48:59,811 DEBUG [RS:1;7b96b36745c6:45609 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/7b96b36745c6:0, corePoolSize=1, maxPoolSize=1 2024-11-21T19:48:59,811 DEBUG [RS:1;7b96b36745c6:45609 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/7b96b36745c6:0, corePoolSize=1, maxPoolSize=1 2024-11-21T19:48:59,811 DEBUG [RS:1;7b96b36745c6:45609 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/7b96b36745c6:0, corePoolSize=1, maxPoolSize=1 2024-11-21T19:48:59,812 DEBUG [RS:1;7b96b36745c6:45609 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/7b96b36745c6:0, corePoolSize=1, maxPoolSize=1 2024-11-21T19:48:59,812 INFO [RS:0;7b96b36745c6:43071 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-21T19:48:59,812 DEBUG [RS:1;7b96b36745c6:45609 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/7b96b36745c6:0, corePoolSize=3, maxPoolSize=3 2024-11-21T19:48:59,812 INFO [RS:0;7b96b36745c6:43071 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-21T19:48:59,812 INFO [RS:0;7b96b36745c6:43071 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-21T19:48:59,812 DEBUG [RS:1;7b96b36745c6:45609 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/7b96b36745c6:0, corePoolSize=3, maxPoolSize=3 2024-11-21T19:48:59,812 INFO [RS:0;7b96b36745c6:43071 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-21T19:48:59,812 INFO [RS:0;7b96b36745c6:43071 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-21T19:48:59,812 INFO [RS:0;7b96b36745c6:43071 {}] hbase.ChoreService(168): Chore ScheduledChore name=7b96b36745c6,43071,1732218539369-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-21T19:48:59,815 INFO [RS:1;7b96b36745c6:45609 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-21T19:48:59,815 INFO [RS:1;7b96b36745c6:45609 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-21T19:48:59,815 INFO [RS:1;7b96b36745c6:45609 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-21T19:48:59,815 INFO [RS:1;7b96b36745c6:45609 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-21T19:48:59,815 INFO [RS:1;7b96b36745c6:45609 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-21T19:48:59,815 INFO [RS:1;7b96b36745c6:45609 {}] hbase.ChoreService(168): Chore ScheduledChore name=7b96b36745c6,45609,1732218539412-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-21T19:48:59,830 INFO [RS:0;7b96b36745c6:43071 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-21T19:48:59,831 INFO [RS:0;7b96b36745c6:43071 {}] hbase.ChoreService(168): Chore ScheduledChore name=7b96b36745c6,43071,1732218539369-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-21T19:48:59,831 INFO [RS:0;7b96b36745c6:43071 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-21T19:48:59,831 INFO [RS:0;7b96b36745c6:43071 {}] regionserver.Replication(171): 7b96b36745c6,43071,1732218539369 started 2024-11-21T19:48:59,833 INFO [RS:1;7b96b36745c6:45609 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-21T19:48:59,833 INFO [RS:1;7b96b36745c6:45609 {}] hbase.ChoreService(168): Chore ScheduledChore name=7b96b36745c6,45609,1732218539412-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-21T19:48:59,833 INFO [RS:1;7b96b36745c6:45609 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-21T19:48:59,833 INFO [RS:1;7b96b36745c6:45609 {}] regionserver.Replication(171): 7b96b36745c6,45609,1732218539412 started 2024-11-21T19:48:59,850 INFO [RS:1;7b96b36745c6:45609 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-21T19:48:59,850 INFO [RS:1;7b96b36745c6:45609 {}] regionserver.HRegionServer(1482): Serving as 7b96b36745c6,45609,1732218539412, RpcServer on 7b96b36745c6/172.17.0.2:45609, sessionid=0x10116d8419f0002 2024-11-21T19:48:59,850 DEBUG [RS:1;7b96b36745c6:45609 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-21T19:48:59,850 DEBUG [RS:1;7b96b36745c6:45609 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 7b96b36745c6,45609,1732218539412 2024-11-21T19:48:59,850 DEBUG [RS:1;7b96b36745c6:45609 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '7b96b36745c6,45609,1732218539412' 2024-11-21T19:48:59,850 DEBUG [RS:1;7b96b36745c6:45609 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-21T19:48:59,851 DEBUG [RS:1;7b96b36745c6:45609 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-21T19:48:59,852 DEBUG [RS:1;7b96b36745c6:45609 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-21T19:48:59,852 DEBUG [RS:1;7b96b36745c6:45609 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-21T19:48:59,852 DEBUG [RS:1;7b96b36745c6:45609 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 7b96b36745c6,45609,1732218539412 2024-11-21T19:48:59,852 DEBUG [RS:1;7b96b36745c6:45609 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '7b96b36745c6,45609,1732218539412' 2024-11-21T19:48:59,852 DEBUG [RS:1;7b96b36745c6:45609 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-21T19:48:59,852 DEBUG [RS:1;7b96b36745c6:45609 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-21T19:48:59,853 INFO [RS:0;7b96b36745c6:43071 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-21T19:48:59,853 DEBUG [RS:1;7b96b36745c6:45609 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-21T19:48:59,853 INFO [RS:1;7b96b36745c6:45609 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-21T19:48:59,853 INFO [RS:1;7b96b36745c6:45609 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-21T19:48:59,853 INFO [RS:0;7b96b36745c6:43071 {}] regionserver.HRegionServer(1482): Serving as 7b96b36745c6,43071,1732218539369, RpcServer on 7b96b36745c6/172.17.0.2:43071, sessionid=0x10116d8419f0001 2024-11-21T19:48:59,853 DEBUG [RS:0;7b96b36745c6:43071 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-21T19:48:59,853 DEBUG [RS:0;7b96b36745c6:43071 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 7b96b36745c6,43071,1732218539369 2024-11-21T19:48:59,853 DEBUG [RS:0;7b96b36745c6:43071 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '7b96b36745c6,43071,1732218539369' 2024-11-21T19:48:59,853 DEBUG [RS:0;7b96b36745c6:43071 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-21T19:48:59,854 DEBUG [RS:0;7b96b36745c6:43071 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-21T19:48:59,854 DEBUG [RS:0;7b96b36745c6:43071 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-21T19:48:59,854 DEBUG [RS:0;7b96b36745c6:43071 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-21T19:48:59,854 DEBUG [RS:0;7b96b36745c6:43071 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 7b96b36745c6,43071,1732218539369 2024-11-21T19:48:59,854 DEBUG [RS:0;7b96b36745c6:43071 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '7b96b36745c6,43071,1732218539369' 2024-11-21T19:48:59,854 DEBUG [RS:0;7b96b36745c6:43071 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-21T19:48:59,855 DEBUG [RS:0;7b96b36745c6:43071 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-21T19:48:59,855 DEBUG [RS:0;7b96b36745c6:43071 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-21T19:48:59,856 INFO [RS:0;7b96b36745c6:43071 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-21T19:48:59,856 INFO [RS:0;7b96b36745c6:43071 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-21T19:48:59,906 WARN [7b96b36745c6:44549 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-21T19:48:59,954 INFO [RS:1;7b96b36745c6:45609 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-21T19:48:59,955 INFO [RS:1;7b96b36745c6:45609 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=7b96b36745c6%2C45609%2C1732218539412, suffix=, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0b774921-3780-c936-4fe1-1d1dcb4ce861/545083c6-d774-d2ca-7409-0fbc2c22cf1a/WALs/7b96b36745c6,45609,1732218539412, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0b774921-3780-c936-4fe1-1d1dcb4ce861/545083c6-d774-d2ca-7409-0fbc2c22cf1a/oldWALs, maxLogs=55 2024-11-21T19:48:59,956 INFO [RS:0;7b96b36745c6:43071 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-21T19:48:59,957 INFO [RS:0;7b96b36745c6:43071 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=7b96b36745c6%2C43071%2C1732218539369, suffix=, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0b774921-3780-c936-4fe1-1d1dcb4ce861/545083c6-d774-d2ca-7409-0fbc2c22cf1a/WALs/7b96b36745c6,43071,1732218539369, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0b774921-3780-c936-4fe1-1d1dcb4ce861/545083c6-d774-d2ca-7409-0fbc2c22cf1a/oldWALs, maxLogs=55 2024-11-21T19:48:59,976 INFO [RS:0;7b96b36745c6:43071 {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0b774921-3780-c936-4fe1-1d1dcb4ce861/545083c6-d774-d2ca-7409-0fbc2c22cf1a/WALs/7b96b36745c6,43071,1732218539369/7b96b36745c6%2C43071%2C1732218539369.1732218539958 2024-11-21T19:48:59,977 DEBUG [RS:0;7b96b36745c6:43071 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-11-21T19:48:59,983 INFO [RS:1;7b96b36745c6:45609 {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0b774921-3780-c936-4fe1-1d1dcb4ce861/545083c6-d774-d2ca-7409-0fbc2c22cf1a/WALs/7b96b36745c6,45609,1732218539412/7b96b36745c6%2C45609%2C1732218539412.1732218539957 2024-11-21T19:48:59,983 DEBUG [RS:1;7b96b36745c6:45609 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-11-21T19:49:00,157 DEBUG [7b96b36745c6:44549 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=2, allServersCount=2 2024-11-21T19:49:00,157 DEBUG [7b96b36745c6:44549 {}] balancer.BalancerClusterState(204): Hosts are {7b96b36745c6=0} racks are {/default-rack=0} 2024-11-21T19:49:00,158 DEBUG [7b96b36745c6:44549 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-11-21T19:49:00,158 DEBUG [7b96b36745c6:44549 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-11-21T19:49:00,158 DEBUG [7b96b36745c6:44549 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-11-21T19:49:00,158 DEBUG [7b96b36745c6:44549 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-11-21T19:49:00,158 INFO [7b96b36745c6:44549 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-11-21T19:49:00,158 INFO [7b96b36745c6:44549 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-11-21T19:49:00,158 DEBUG [7b96b36745c6:44549 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-11-21T19:49:00,159 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=7b96b36745c6,45609,1732218539412 2024-11-21T19:49:00,160 INFO [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 7b96b36745c6,45609,1732218539412, state=OPENING 2024-11-21T19:49:00,162 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2024-11-21T19:49:00,163 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45609-0x10116d8419f0002, quorum=127.0.0.1:54341, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-21T19:49:00,163 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44549-0x10116d8419f0000, quorum=127.0.0.1:54341, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-21T19:49:00,163 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43071-0x10116d8419f0001, quorum=127.0.0.1:54341, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-21T19:49:00,164 DEBUG [PEWorker-3 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-21T19:49:00,164 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-21T19:49:00,164 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-21T19:49:00,164 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=7b96b36745c6,45609,1732218539412}] 2024-11-21T19:49:00,164 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-21T19:49:00,317 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-21T19:49:00,320 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-8-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60193, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-21T19:49:00,321 INFO [RS_OPEN_META-regionserver/7b96b36745c6:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-11-21T19:49:00,322 INFO [RS_OPEN_META-regionserver/7b96b36745c6:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-21T19:49:00,322 INFO [RS_OPEN_META-regionserver/7b96b36745c6:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-11-21T19:49:00,323 INFO [RS_OPEN_META-regionserver/7b96b36745c6:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=7b96b36745c6%2C45609%2C1732218539412.meta, suffix=.meta, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0b774921-3780-c936-4fe1-1d1dcb4ce861/545083c6-d774-d2ca-7409-0fbc2c22cf1a/WALs/7b96b36745c6,45609,1732218539412, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0b774921-3780-c936-4fe1-1d1dcb4ce861/545083c6-d774-d2ca-7409-0fbc2c22cf1a/oldWALs, maxLogs=55 2024-11-21T19:49:00,351 INFO [RS_OPEN_META-regionserver/7b96b36745c6:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0b774921-3780-c936-4fe1-1d1dcb4ce861/545083c6-d774-d2ca-7409-0fbc2c22cf1a/WALs/7b96b36745c6,45609,1732218539412/7b96b36745c6%2C45609%2C1732218539412.meta.1732218540325.meta 2024-11-21T19:49:00,352 DEBUG [RS_OPEN_META-regionserver/7b96b36745c6:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-11-21T19:49:00,352 DEBUG [RS_OPEN_META-regionserver/7b96b36745c6:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-11-21T19:49:00,352 DEBUG [RS_OPEN_META-regionserver/7b96b36745c6:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-11-21T19:49:00,352 DEBUG [RS_OPEN_META-regionserver/7b96b36745c6:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-11-21T19:49:00,353 INFO [RS_OPEN_META-regionserver/7b96b36745c6:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-11-21T19:49:00,353 DEBUG [RS_OPEN_META-regionserver/7b96b36745c6:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-11-21T19:49:00,353 DEBUG [RS_OPEN_META-regionserver/7b96b36745c6:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-21T19:49:00,353 DEBUG [RS_OPEN_META-regionserver/7b96b36745c6:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-11-21T19:49:00,353 DEBUG [RS_OPEN_META-regionserver/7b96b36745c6:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-11-21T19:49:00,354 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family info of region 1588230740 2024-11-21T19:49:00,355 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-21T19:49:00,355 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T19:49:00,355 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-21T19:49:00,356 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family ns of region 1588230740 2024-11-21T19:49:00,356 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-21T19:49:00,356 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T19:49:00,356 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-21T19:49:00,356 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family rep_barrier of region 1588230740 2024-11-21T19:49:00,357 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-21T19:49:00,357 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T19:49:00,357 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-21T19:49:00,357 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family table of region 1588230740 2024-11-21T19:49:00,358 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-21T19:49:00,358 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T19:49:00,358 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-21T19:49:00,358 DEBUG [RS_OPEN_META-regionserver/7b96b36745c6:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-21T19:49:00,359 DEBUG [RS_OPEN_META-regionserver/7b96b36745c6:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0b774921-3780-c936-4fe1-1d1dcb4ce861/545083c6-d774-d2ca-7409-0fbc2c22cf1a/data/hbase/meta/1588230740 2024-11-21T19:49:00,360 DEBUG [RS_OPEN_META-regionserver/7b96b36745c6:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0b774921-3780-c936-4fe1-1d1dcb4ce861/545083c6-d774-d2ca-7409-0fbc2c22cf1a/data/hbase/meta/1588230740 2024-11-21T19:49:00,360 DEBUG [RS_OPEN_META-regionserver/7b96b36745c6:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-21T19:49:00,360 DEBUG [RS_OPEN_META-regionserver/7b96b36745c6:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-21T19:49:00,360 DEBUG [RS_OPEN_META-regionserver/7b96b36745c6:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-21T19:49:00,361 DEBUG [RS_OPEN_META-regionserver/7b96b36745c6:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-21T19:49:00,361 INFO [RS_OPEN_META-regionserver/7b96b36745c6:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=74151726, jitterRate=0.10494682192802429}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-21T19:49:00,361 DEBUG [RS_OPEN_META-regionserver/7b96b36745c6:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-11-21T19:49:00,362 DEBUG [RS_OPEN_META-regionserver/7b96b36745c6:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1732218540353Writing region info on filesystem at 1732218540354 (+1 ms)Initializing all the Stores at 1732218540354Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732218540354Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732218540354Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732218540354Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732218540354Cleaning up temporary data from old regions at 1732218540360 (+6 ms)Running coprocessor post-open hooks at 1732218540361 (+1 ms)Region opened successfully at 1732218540362 (+1 ms) 2024-11-21T19:49:00,363 INFO [RS_OPEN_META-regionserver/7b96b36745c6:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1732218540317 2024-11-21T19:49:00,365 DEBUG [RS_OPEN_META-regionserver/7b96b36745c6:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-11-21T19:49:00,365 INFO [RS_OPEN_META-regionserver/7b96b36745c6:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-11-21T19:49:00,366 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=7b96b36745c6,45609,1732218539412 2024-11-21T19:49:00,366 INFO [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 7b96b36745c6,45609,1732218539412, state=OPEN 2024-11-21T19:49:00,368 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43071-0x10116d8419f0001, quorum=127.0.0.1:54341, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-21T19:49:00,368 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45609-0x10116d8419f0002, quorum=127.0.0.1:54341, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-21T19:49:00,369 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44549-0x10116d8419f0000, quorum=127.0.0.1:54341, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-21T19:49:00,369 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=7b96b36745c6,45609,1732218539412 2024-11-21T19:49:00,369 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-21T19:49:00,369 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-21T19:49:00,369 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-21T19:49:00,371 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2 2024-11-21T19:49:00,371 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=7b96b36745c6,45609,1732218539412 in 205 msec 2024-11-21T19:49:00,372 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1 2024-11-21T19:49:00,372 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 619 msec 2024-11-21T19:49:00,373 DEBUG [PEWorker-2 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-21T19:49:00,373 INFO [PEWorker-2 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces 2024-11-21T19:49:00,378 DEBUG [PEWorker-2 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-21T19:49:00,378 DEBUG [PEWorker-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=7b96b36745c6,45609,1732218539412, seqNum=-1] 2024-11-21T19:49:00,378 DEBUG [PEWorker-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-21T19:49:00,380 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-8-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57051, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-21T19:49:00,387 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 714 msec 2024-11-21T19:49:00,387 INFO [master/7b96b36745c6:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1732218540387, completionTime=-1 2024-11-21T19:49:00,387 INFO [master/7b96b36745c6:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=2; waited=0ms, expected min=2 server(s), max=2 server(s), master is running 2024-11-21T19:49:00,387 DEBUG [master/7b96b36745c6:0:becomeActiveMaster {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-21T19:49:00,389 INFO [master/7b96b36745c6:0:becomeActiveMaster {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-11-21T19:49:00,390 INFO [master/7b96b36745c6:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732218600389 2024-11-21T19:49:00,390 INFO [master/7b96b36745c6:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732218660390 2024-11-21T19:49:00,390 INFO [master/7b96b36745c6:0:becomeActiveMaster {}] assignment.AssignmentManager(1783): Joined the cluster in 2 msec 2024-11-21T19:49:00,390 INFO [master/7b96b36745c6:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=7b96b36745c6,44549,1732218539289-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-21T19:49:00,390 INFO [master/7b96b36745c6:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=7b96b36745c6,44549,1732218539289-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-21T19:49:00,390 INFO [master/7b96b36745c6:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=7b96b36745c6,44549,1732218539289-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-21T19:49:00,390 INFO [master/7b96b36745c6:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-7b96b36745c6:44549, period=300000, unit=MILLISECONDS is enabled. 2024-11-21T19:49:00,390 INFO [master/7b96b36745c6:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2024-11-21T19:49:00,391 INFO [master/7b96b36745c6:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled. 2024-11-21T19:49:00,394 DEBUG [master/7b96b36745c6:0.Chore.1 {}] janitor.CatalogJanitor(180): 2024-11-21T19:49:00,398 INFO [master/7b96b36745c6:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 0.923sec 2024-11-21T19:49:00,398 INFO [master/7b96b36745c6:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2024-11-21T19:49:00,398 INFO [master/7b96b36745c6:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2024-11-21T19:49:00,398 INFO [master/7b96b36745c6:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2024-11-21T19:49:00,398 INFO [master/7b96b36745c6:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2024-11-21T19:49:00,398 INFO [master/7b96b36745c6:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2024-11-21T19:49:00,398 INFO [master/7b96b36745c6:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=7b96b36745c6,44549,1732218539289-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-21T19:49:00,398 INFO [master/7b96b36745c6:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=7b96b36745c6,44549,1732218539289-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2024-11-21T19:49:00,399 DEBUG [master/7b96b36745c6:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds 2024-11-21T19:49:00,400 INFO [master/7b96b36745c6:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2024-11-21T19:49:00,400 INFO [master/7b96b36745c6:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=7b96b36745c6,44549,1732218539289-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-21T19:49:00,450 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@284befa8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T19:49:00,451 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 7b96b36745c6,44549,-1 for getting cluster id 2024-11-21T19:49:00,451 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-21T19:49:00,453 DEBUG [HMaster-EventLoopGroup-6-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f26c0bbc-ffc3-4204-8c91-2c4d735cdc03' 2024-11-21T19:49:00,453 DEBUG [RPCClient-NioEventLoopGroup-5-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-21T19:49:00,453 DEBUG [RPCClient-NioEventLoopGroup-5-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f26c0bbc-ffc3-4204-8c91-2c4d735cdc03" 2024-11-21T19:49:00,454 DEBUG [RPCClient-NioEventLoopGroup-5-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@32c38e10, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T19:49:00,454 DEBUG [RPCClient-NioEventLoopGroup-5-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [7b96b36745c6,44549,-1] 2024-11-21T19:49:00,454 DEBUG [RPCClient-NioEventLoopGroup-5-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-21T19:49:00,455 DEBUG [RPCClient-NioEventLoopGroup-5-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T19:49:00,457 INFO [HMaster-EventLoopGroup-6-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47888, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-21T19:49:00,458 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2beb4aff, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T19:49:00,459 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-21T19:49:00,460 DEBUG [RPCClient-NioEventLoopGroup-5-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=7b96b36745c6,45609,1732218539412, seqNum=-1] 2024-11-21T19:49:00,460 DEBUG [RPCClient-NioEventLoopGroup-5-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-21T19:49:00,463 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-8-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38644, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-21T19:49:00,465 INFO [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=7b96b36745c6,44549,1732218539289 2024-11-21T19:49:00,466 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-21T19:49:00,467 DEBUG [RPCClient-NioEventLoopGroup-5-6 {}] client.AsyncConnectionImpl(321): The fetched master address is 7b96b36745c6,44549,1732218539289 2024-11-21T19:49:00,467 DEBUG [RPCClient-NioEventLoopGroup-5-6 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@19b512d4 2024-11-21T19:49:00,467 DEBUG [RPCClient-NioEventLoopGroup-5-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-21T19:49:00,469 INFO [HMaster-EventLoopGroup-6-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47894, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-21T19:49:00,470 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44549 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'testBlockEvictionOnGracefulStop1732218540465', {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-21T19:49:00,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44549 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=testBlockEvictionOnGracefulStop1732218540465 2024-11-21T19:49:00,471 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=testBlockEvictionOnGracefulStop1732218540465 execute state=CREATE_TABLE_PRE_OPERATION 2024-11-21T19:49:00,472 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T19:49:00,472 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44549 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "testBlockEvictionOnGracefulStop1732218540465" procId is: 4 2024-11-21T19:49:00,472 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=testBlockEvictionOnGracefulStop1732218540465 execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-21T19:49:00,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44549 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-21T19:49:00,487 INFO [RegionOpenAndInit-testBlockEvictionOnGracefulStop1732218540465-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 49a51e328dbf56d3b52aa1d341d17e4e, NAME => 'testBlockEvictionOnGracefulStop1732218540465,,1732218540469.49a51e328dbf56d3b52aa1d341d17e4e.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testBlockEvictionOnGracefulStop1732218540465', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0b774921-3780-c936-4fe1-1d1dcb4ce861/545083c6-d774-d2ca-7409-0fbc2c22cf1a 2024-11-21T19:49:00,494 DEBUG [RegionOpenAndInit-testBlockEvictionOnGracefulStop1732218540465-pool-0 {}] regionserver.HRegion(898): Instantiated testBlockEvictionOnGracefulStop1732218540465,,1732218540469.49a51e328dbf56d3b52aa1d341d17e4e.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-21T19:49:00,494 DEBUG [RegionOpenAndInit-testBlockEvictionOnGracefulStop1732218540465-pool-0 {}] regionserver.HRegion(1722): Closing 49a51e328dbf56d3b52aa1d341d17e4e, disabling compactions & flushes 2024-11-21T19:49:00,494 INFO [RegionOpenAndInit-testBlockEvictionOnGracefulStop1732218540465-pool-0 {}] regionserver.HRegion(1755): Closing region testBlockEvictionOnGracefulStop1732218540465,,1732218540469.49a51e328dbf56d3b52aa1d341d17e4e. 2024-11-21T19:49:00,494 DEBUG [RegionOpenAndInit-testBlockEvictionOnGracefulStop1732218540465-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on testBlockEvictionOnGracefulStop1732218540465,,1732218540469.49a51e328dbf56d3b52aa1d341d17e4e. 2024-11-21T19:49:00,494 DEBUG [RegionOpenAndInit-testBlockEvictionOnGracefulStop1732218540465-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on testBlockEvictionOnGracefulStop1732218540465,,1732218540469.49a51e328dbf56d3b52aa1d341d17e4e. after waiting 0 ms 2024-11-21T19:49:00,494 DEBUG [RegionOpenAndInit-testBlockEvictionOnGracefulStop1732218540465-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region testBlockEvictionOnGracefulStop1732218540465,,1732218540469.49a51e328dbf56d3b52aa1d341d17e4e. 2024-11-21T19:49:00,494 INFO [RegionOpenAndInit-testBlockEvictionOnGracefulStop1732218540465-pool-0 {}] regionserver.HRegion(1973): Closed testBlockEvictionOnGracefulStop1732218540465,,1732218540469.49a51e328dbf56d3b52aa1d341d17e4e. 2024-11-21T19:49:00,494 DEBUG [RegionOpenAndInit-testBlockEvictionOnGracefulStop1732218540465-pool-0 {}] regionserver.HRegion(1676): Region close journal for 49a51e328dbf56d3b52aa1d341d17e4e: Waiting for close lock at 1732218540494Disabling compacts and flushes for region at 1732218540494Disabling writes for close at 1732218540494Writing region close event to WAL at 1732218540494Closed at 1732218540494 2024-11-21T19:49:00,495 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=testBlockEvictionOnGracefulStop1732218540465 execute state=CREATE_TABLE_ADD_TO_META 2024-11-21T19:49:00,495 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testBlockEvictionOnGracefulStop1732218540465,,1732218540469.49a51e328dbf56d3b52aa1d341d17e4e.","families":{"info":[{"qualifier":"regioninfo","vlen":78,"tag":[],"timestamp":"1732218540495"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732218540495"}]},"ts":"1732218540495"} 2024-11-21T19:49:00,498 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-21T19:49:00,498 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=testBlockEvictionOnGracefulStop1732218540465 execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-21T19:49:00,499 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testBlockEvictionOnGracefulStop1732218540465","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732218540498"}]},"ts":"1732218540498"} 2024-11-21T19:49:00,501 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=testBlockEvictionOnGracefulStop1732218540465, state=ENABLING in hbase:meta 2024-11-21T19:49:00,501 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(204): Hosts are {7b96b36745c6=0} racks are {/default-rack=0} 2024-11-21T19:49:00,501 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-11-21T19:49:00,501 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-11-21T19:49:00,502 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-11-21T19:49:00,502 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-11-21T19:49:00,502 INFO [PEWorker-3 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-11-21T19:49:00,502 INFO [PEWorker-3 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-11-21T19:49:00,502 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-11-21T19:49:00,502 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1732218540465, region=49a51e328dbf56d3b52aa1d341d17e4e, ASSIGN}] 2024-11-21T19:49:00,503 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1732218540465, region=49a51e328dbf56d3b52aa1d341d17e4e, ASSIGN 2024-11-21T19:49:00,504 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1732218540465, region=49a51e328dbf56d3b52aa1d341d17e4e, ASSIGN; state=OFFLINE, location=7b96b36745c6,45609,1732218539412; forceNewPlan=false, retain=false 2024-11-21T19:49:00,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44549 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-21T19:49:00,654 INFO [7b96b36745c6:44549 {}] balancer.BaseLoadBalancer(388): Reassigned 1 regions. 1 retained the pre-restart assignment. 2024-11-21T19:49:00,655 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=49a51e328dbf56d3b52aa1d341d17e4e, regionState=OPENING, regionLocation=7b96b36745c6,45609,1732218539412 2024-11-21T19:49:00,658 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-8-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1732218540465, region=49a51e328dbf56d3b52aa1d341d17e4e, ASSIGN because future has completed 2024-11-21T19:49:00,658 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 49a51e328dbf56d3b52aa1d341d17e4e, server=7b96b36745c6,45609,1732218539412}] 2024-11-21T19:49:00,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44549 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-21T19:49:00,813 INFO [RS_OPEN_REGION-regionserver/7b96b36745c6:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(132): Open testBlockEvictionOnGracefulStop1732218540465,,1732218540469.49a51e328dbf56d3b52aa1d341d17e4e. 2024-11-21T19:49:00,813 DEBUG [RS_OPEN_REGION-regionserver/7b96b36745c6:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7752): Opening region: {ENCODED => 49a51e328dbf56d3b52aa1d341d17e4e, NAME => 'testBlockEvictionOnGracefulStop1732218540465,,1732218540469.49a51e328dbf56d3b52aa1d341d17e4e.', STARTKEY => '', ENDKEY => ''} 2024-11-21T19:49:00,814 DEBUG [RS_OPEN_REGION-regionserver/7b96b36745c6:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testBlockEvictionOnGracefulStop1732218540465 49a51e328dbf56d3b52aa1d341d17e4e 2024-11-21T19:49:00,814 DEBUG [RS_OPEN_REGION-regionserver/7b96b36745c6:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(898): Instantiated testBlockEvictionOnGracefulStop1732218540465,,1732218540469.49a51e328dbf56d3b52aa1d341d17e4e.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-21T19:49:00,814 DEBUG [RS_OPEN_REGION-regionserver/7b96b36745c6:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7794): checking encryption for 49a51e328dbf56d3b52aa1d341d17e4e 2024-11-21T19:49:00,814 DEBUG [RS_OPEN_REGION-regionserver/7b96b36745c6:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7797): checking classloading for 49a51e328dbf56d3b52aa1d341d17e4e 2024-11-21T19:49:00,815 INFO [StoreOpener-49a51e328dbf56d3b52aa1d341d17e4e-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family family of region 49a51e328dbf56d3b52aa1d341d17e4e 2024-11-21T19:49:00,819 INFO [StoreOpener-49a51e328dbf56d3b52aa1d341d17e4e-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 49a51e328dbf56d3b52aa1d341d17e4e columnFamilyName family 2024-11-21T19:49:00,819 DEBUG [StoreOpener-49a51e328dbf56d3b52aa1d341d17e4e-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T19:49:00,819 INFO [StoreOpener-49a51e328dbf56d3b52aa1d341d17e4e-1 {}] regionserver.HStore(327): Store=49a51e328dbf56d3b52aa1d341d17e4e/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-21T19:49:00,820 DEBUG [RS_OPEN_REGION-regionserver/7b96b36745c6:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1038): replaying wal for 49a51e328dbf56d3b52aa1d341d17e4e 2024-11-21T19:49:00,821 DEBUG [RS_OPEN_REGION-regionserver/7b96b36745c6:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0b774921-3780-c936-4fe1-1d1dcb4ce861/545083c6-d774-d2ca-7409-0fbc2c22cf1a/data/default/testBlockEvictionOnGracefulStop1732218540465/49a51e328dbf56d3b52aa1d341d17e4e 2024-11-21T19:49:00,821 DEBUG [RS_OPEN_REGION-regionserver/7b96b36745c6:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0b774921-3780-c936-4fe1-1d1dcb4ce861/545083c6-d774-d2ca-7409-0fbc2c22cf1a/data/default/testBlockEvictionOnGracefulStop1732218540465/49a51e328dbf56d3b52aa1d341d17e4e 2024-11-21T19:49:00,821 DEBUG [RS_OPEN_REGION-regionserver/7b96b36745c6:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1048): stopping wal replay for 49a51e328dbf56d3b52aa1d341d17e4e 2024-11-21T19:49:00,821 DEBUG [RS_OPEN_REGION-regionserver/7b96b36745c6:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1060): Cleaning up temporary data for 49a51e328dbf56d3b52aa1d341d17e4e 2024-11-21T19:49:00,822 DEBUG [RS_OPEN_REGION-regionserver/7b96b36745c6:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1093): writing seq id for 49a51e328dbf56d3b52aa1d341d17e4e 2024-11-21T19:49:00,828 DEBUG [RS_OPEN_REGION-regionserver/7b96b36745c6:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0b774921-3780-c936-4fe1-1d1dcb4ce861/545083c6-d774-d2ca-7409-0fbc2c22cf1a/data/default/testBlockEvictionOnGracefulStop1732218540465/49a51e328dbf56d3b52aa1d341d17e4e/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-21T19:49:00,828 INFO [RS_OPEN_REGION-regionserver/7b96b36745c6:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1114): Opened 49a51e328dbf56d3b52aa1d341d17e4e; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=66149303, jitterRate=-0.014298573136329651}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-21T19:49:00,828 DEBUG [RS_OPEN_REGION-regionserver/7b96b36745c6:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 49a51e328dbf56d3b52aa1d341d17e4e 2024-11-21T19:49:00,829 DEBUG [RS_OPEN_REGION-regionserver/7b96b36745c6:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1006): Region open journal for 49a51e328dbf56d3b52aa1d341d17e4e: Running coprocessor pre-open hook at 1732218540814Writing region info on filesystem at 1732218540814Initializing all the Stores at 1732218540814Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732218540814Cleaning up temporary data from old regions at 1732218540821 (+7 ms)Running coprocessor post-open hooks at 1732218540828 (+7 ms)Region opened successfully at 1732218540829 (+1 ms) 2024-11-21T19:49:00,830 INFO [RS_OPEN_REGION-regionserver/7b96b36745c6:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2236): Post open deploy tasks for testBlockEvictionOnGracefulStop1732218540465,,1732218540469.49a51e328dbf56d3b52aa1d341d17e4e., pid=6, masterSystemTime=1732218540811 2024-11-21T19:49:00,832 DEBUG [RS_OPEN_REGION-regionserver/7b96b36745c6:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2266): Finished post open deploy task for testBlockEvictionOnGracefulStop1732218540465,,1732218540469.49a51e328dbf56d3b52aa1d341d17e4e. 2024-11-21T19:49:00,832 INFO [RS_OPEN_REGION-regionserver/7b96b36745c6:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(153): Opened testBlockEvictionOnGracefulStop1732218540465,,1732218540469.49a51e328dbf56d3b52aa1d341d17e4e. 2024-11-21T19:49:00,832 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=49a51e328dbf56d3b52aa1d341d17e4e, regionState=OPEN, openSeqNum=2, regionLocation=7b96b36745c6,45609,1732218539412 2024-11-21T19:49:00,835 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-8-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 49a51e328dbf56d3b52aa1d341d17e4e, server=7b96b36745c6,45609,1732218539412 because future has completed 2024-11-21T19:49:00,838 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=6, resume processing ppid=5 2024-11-21T19:49:00,838 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure 49a51e328dbf56d3b52aa1d341d17e4e, server=7b96b36745c6,45609,1732218539412 in 178 msec 2024-11-21T19:49:00,840 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=5, resume processing ppid=4 2024-11-21T19:49:00,840 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1732218540465, region=49a51e328dbf56d3b52aa1d341d17e4e, ASSIGN in 336 msec 2024-11-21T19:49:00,841 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=testBlockEvictionOnGracefulStop1732218540465 execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-21T19:49:00,841 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testBlockEvictionOnGracefulStop1732218540465","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732218540841"}]},"ts":"1732218540841"} 2024-11-21T19:49:00,843 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=testBlockEvictionOnGracefulStop1732218540465, state=ENABLED in hbase:meta 2024-11-21T19:49:00,843 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=testBlockEvictionOnGracefulStop1732218540465 execute state=CREATE_TABLE_POST_OPERATION 2024-11-21T19:49:00,845 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=testBlockEvictionOnGracefulStop1732218540465 in 374 msec 2024-11-21T19:49:01,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44549 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-21T19:49:01,097 INFO [RPCClient-NioEventLoopGroup-5-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:testBlockEvictionOnGracefulStop1732218540465 completed 2024-11-21T19:49:01,097 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table testBlockEvictionOnGracefulStop1732218540465 get assigned. Timeout = 60000ms 2024-11-21T19:49:01,097 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-21T19:49:01,101 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table testBlockEvictionOnGracefulStop1732218540465 assigned to meta. Checking AM states. 2024-11-21T19:49:01,101 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-21T19:49:01,101 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table testBlockEvictionOnGracefulStop1732218540465 assigned. 2024-11-21T19:49:01,104 DEBUG [RPCClient-NioEventLoopGroup-5-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testBlockEvictionOnGracefulStop1732218540465', row='row1', locateType=CURRENT is [region=testBlockEvictionOnGracefulStop1732218540465,,1732218540469.49a51e328dbf56d3b52aa1d341d17e4e., hostname=7b96b36745c6,45609,1732218539412, seqNum=2] 2024-11-21T19:49:01,112 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 49a51e328dbf56d3b52aa1d341d17e4e 1/1 column families, dataSize=78 B heapSize=480 B 2024-11-21T19:49:01,123 DEBUG [Time-limited test {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on da9e36673a60423a945995608d46d07f 2024-11-21T19:49:01,136 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0b774921-3780-c936-4fe1-1d1dcb4ce861/545083c6-d774-d2ca-7409-0fbc2c22cf1a/data/default/testBlockEvictionOnGracefulStop1732218540465/49a51e328dbf56d3b52aa1d341d17e4e/.tmp/family/da9e36673a60423a945995608d46d07f is 43, key is row1/family:qf1/1/Put/seqid=0 2024-11-21T19:49:01,136 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=78 B at sequenceid=6 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0b774921-3780-c936-4fe1-1d1dcb4ce861/545083c6-d774-d2ca-7409-0fbc2c22cf1a/data/default/testBlockEvictionOnGracefulStop1732218540465/49a51e328dbf56d3b52aa1d341d17e4e/.tmp/family/da9e36673a60423a945995608d46d07f 2024-11-21T19:49:01,139 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0b774921-3780-c936-4fe1-1d1dcb4ce861/545083c6-d774-d2ca-7409-0fbc2c22cf1a/data/default/testBlockEvictionOnGracefulStop1732218540465/49a51e328dbf56d3b52aa1d341d17e4e/.tmp/family/da9e36673a60423a945995608d46d07f as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0b774921-3780-c936-4fe1-1d1dcb4ce861/545083c6-d774-d2ca-7409-0fbc2c22cf1a/data/default/testBlockEvictionOnGracefulStop1732218540465/49a51e328dbf56d3b52aa1d341d17e4e/family/da9e36673a60423a945995608d46d07f 2024-11-21T19:49:01,140 DEBUG [Time-limited test {}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0b774921-3780-c936-4fe1-1d1dcb4ce861/545083c6-d774-d2ca-7409-0fbc2c22cf1a/data/default/testBlockEvictionOnGracefulStop1732218540465/49a51e328dbf56d3b52aa1d341d17e4e/family/da9e36673a60423a945995608d46d07f, delay=915 ms 2024-11-21T19:49:01,141 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0b774921-3780-c936-4fe1-1d1dcb4ce861/545083c6-d774-d2ca-7409-0fbc2c22cf1a/data/default/testBlockEvictionOnGracefulStop1732218540465/49a51e328dbf56d3b52aa1d341d17e4e/family/da9e36673a60423a945995608d46d07f, entries=2, sequenceid=6, filesize=4.9 K 2024-11-21T19:49:01,142 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~78 B/78, heapSize ~464 B/464, currentSize=0 B/0 for 49a51e328dbf56d3b52aa1d341d17e4e in 30ms, sequenceid=6, compaction requested=false 2024-11-21T19:49:01,142 DEBUG [Time-limited test {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'testBlockEvictionOnGracefulStop1732218540465' 2024-11-21T19:49:01,143 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for 49a51e328dbf56d3b52aa1d341d17e4e: 2024-11-21T19:49:01,375 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-21T19:49:02,056 DEBUG [hfile-prefetch-1732218538636 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key da9e36673a60423a945995608d46d07f_0. 2024-11-21T19:49:02,057 DEBUG [hfile-prefetch-1732218538636 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key da9e36673a60423a945995608d46d07f_117. 2024-11-21T19:49:02,057 DEBUG [hfile-prefetch-1732218538636 {}] bucket.BucketCache(2281): Notifying caching completed for file file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0b774921-3780-c936-4fe1-1d1dcb4ce861/545083c6-d774-d2ca-7409-0fbc2c22cf1a/data/default/testBlockEvictionOnGracefulStop1732218540465/49a51e328dbf56d3b52aa1d341d17e4e/family/da9e36673a60423a945995608d46d07f, with total blocks 2, and data blocks 1 2024-11-21T19:49:02,057 DEBUG [hfile-prefetch-1732218538636 {}] bucket.BucketCache(2285): iterating over 0 entries in the backing map 2024-11-21T19:49:02,057 DEBUG [hfile-prefetch-1732218538636 {}] bucket.BucketCache(2308): Prefetch executor completed for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0b774921-3780-c936-4fe1-1d1dcb4ce861/545083c6-d774-d2ca-7409-0fbc2c22cf1a/data/default/testBlockEvictionOnGracefulStop1732218540465/49a51e328dbf56d3b52aa1d341d17e4e/family/da9e36673a60423a945995608d46d07f, but only 0 data blocks were cached. Total data blocks for file: 1. Checking for blocks pending cache in cache writer queue. 2024-11-21T19:49:02,057 DEBUG [hfile-prefetch-1732218538636 {}] bucket.BucketCache(2318): There are still blocks pending caching for file file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0b774921-3780-c936-4fe1-1d1dcb4ce861/545083c6-d774-d2ca-7409-0fbc2c22cf1a/data/default/testBlockEvictionOnGracefulStop1732218540465/49a51e328dbf56d3b52aa1d341d17e4e/family/da9e36673a60423a945995608d46d07f. Will sleep 100ms and try the verification again. 2024-11-21T19:49:02,084 DEBUG [bucket-cache-persister {}] bucket.BucketCachePersister(47): Cache is inconsistent, persisting to disk 2024-11-21T19:49:02,084 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1487): Thread bucket-cache-persister started persisting bucket cache to file 2024-11-21T19:49:02,084 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1494): Persist in new chunked persistence format. 2024-11-21T19:49:02,084 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1673): persistToFile: before persisting backing map size: 0, fullycachedFiles size: 0, chunkSize: 10000 2024-11-21T19:49:02,087 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1680): persistToFile: after persisting backing map size: 0, fullycachedFiles size: 0 2024-11-21T19:49:02,087 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1498): PersistToFile: after persisting backing map size: 0, fullycachedFiles size: 0, file name: bucket.persistence1732218542084 2024-11-21T19:49:02,087 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1509): Thread bucket-cache-persister finished persisting bucket cache to file, renaming 2024-11-21T19:49:02,143 INFO [Time-limited test {}] hbase.SingleProcessHBaseCluster(504): Stopping Thread[RS:1;7b96b36745c6:45609,5,FailOnTimeoutGroup] 2024-11-21T19:49:02,144 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '7b96b36745c6,45609,1732218539412' ***** 2024-11-21T19:49:02,144 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Stopping rs 1 2024-11-21T19:49:02,144 INFO [RS:1;7b96b36745c6:45609 {}] bucket.BucketCache(1738): Disabling cache 2024-11-21T19:49:02,144 INFO [RS:1;7b96b36745c6:45609 {}] bucket.BucketCache(1762): Shutdown bucket cache: IO persistent=true; path to write=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/11a8a75e-d82a-1368-d32b-668baa66865a/bucket.persistence 2024-11-21T19:49:02,144 INFO [Time-limited test-BucketCacheWriter-1 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-1 exiting, cacheEnabled=false 2024-11-21T19:49:02,144 INFO [Time-limited test-BucketCacheWriter-2 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-2 exiting, cacheEnabled=false 2024-11-21T19:49:02,144 INFO [Time-limited test-BucketCacheWriter-0 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-0 exiting, cacheEnabled=false 2024-11-21T19:49:02,144 INFO [RS:1;7b96b36745c6:45609 {}] bucket.BucketCache(1768): Shutting down cache persister thread. 2024-11-21T19:49:02,144 WARN [bucket-cache-persister {}] bucket.BucketCachePersister(64): Interrupting BucketCachePersister thread. java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCachePersister.run(BucketCachePersister.java:45) ~[classes/:?] 2024-11-21T19:49:02,154 DEBUG [RS:1;7b96b36745c6:45609 {}] bucket.BucketCache(1487): Thread RS:1;7b96b36745c6:45609 started persisting bucket cache to file 2024-11-21T19:49:02,154 DEBUG [RS:1;7b96b36745c6:45609 {}] bucket.BucketCache(1494): Persist in new chunked persistence format. 2024-11-21T19:49:02,155 DEBUG [RS:1;7b96b36745c6:45609 {}] bucket.BucketCache(1673): persistToFile: before persisting backing map size: 1, fullycachedFiles size: 0, chunkSize: 10000 2024-11-21T19:49:02,157 DEBUG [hfile-prefetch-1732218538636 {}] bucket.BucketCache(2281): Notifying caching completed for file file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0b774921-3780-c936-4fe1-1d1dcb4ce861/545083c6-d774-d2ca-7409-0fbc2c22cf1a/data/default/testBlockEvictionOnGracefulStop1732218540465/49a51e328dbf56d3b52aa1d341d17e4e/family/da9e36673a60423a945995608d46d07f, with total blocks 2, and data blocks 1 2024-11-21T19:49:02,157 DEBUG [hfile-prefetch-1732218538636 {}] bucket.BucketCache(2285): iterating over 1 entries in the backing map 2024-11-21T19:49:02,158 DEBUG [hfile-prefetch-1732218538636 {}] bucket.BucketCache(2293): found block for file da9e36673a60423a945995608d46d07f in the backing map. Acquiring read lock for offset 0 2024-11-21T19:49:02,158 DEBUG [hfile-prefetch-1732218538636 {}] bucket.BucketCache(2305): File file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0b774921-3780-c936-4fe1-1d1dcb4ce861/545083c6-d774-d2ca-7409-0fbc2c22cf1a/data/default/testBlockEvictionOnGracefulStop1732218540465/49a51e328dbf56d3b52aa1d341d17e4e/family/da9e36673a60423a945995608d46d07f has now been fully cached. 2024-11-21T19:49:02,158 DEBUG [hfile-prefetch-1732218538636 {}] hfile.PrefetchExecutor(121): Prefetch completed for da9e36673a60423a945995608d46d07f 2024-11-21T19:49:02,162 DEBUG [RS:1;7b96b36745c6:45609 {}] bucket.BucketCache(1680): persistToFile: after persisting backing map size: 1, fullycachedFiles size: 1 2024-11-21T19:49:02,162 DEBUG [RS:1;7b96b36745c6:45609 {}] bucket.BucketCache(1498): PersistToFile: after persisting backing map size: 1, fullycachedFiles size: 1, file name: bucket.persistence1732218542154 2024-11-21T19:49:02,162 DEBUG [RS:1;7b96b36745c6:45609 {}] bucket.BucketCache(1509): Thread RS:1;7b96b36745c6:45609 finished persisting bucket cache to file, renaming 2024-11-21T19:49:02,162 INFO [RS:1;7b96b36745c6:45609 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-21T19:49:02,162 INFO [RS:1;7b96b36745c6:45609 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-21T19:49:02,162 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-21T19:49:02,162 INFO [RS:1;7b96b36745c6:45609 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-21T19:49:02,162 INFO [RS:1;7b96b36745c6:45609 {}] regionserver.HRegionServer(3091): Received CLOSE for 49a51e328dbf56d3b52aa1d341d17e4e 2024-11-21T19:49:02,162 INFO [RS:1;7b96b36745c6:45609 {}] regionserver.HRegionServer(959): stopping server 7b96b36745c6,45609,1732218539412 2024-11-21T19:49:02,163 INFO [RS:1;7b96b36745c6:45609 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-21T19:49:02,163 INFO [RS:1;7b96b36745c6:45609 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:1;7b96b36745c6:45609. 2024-11-21T19:49:02,163 INFO [RS_CLOSE_REGION-regionserver/7b96b36745c6:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1662): Closing region testBlockEvictionOnGracefulStop1732218540465,,1732218540469.49a51e328dbf56d3b52aa1d341d17e4e. during a graceful stop, and cache persistence is on, so setting evict on close to false. 2024-11-21T19:49:02,163 DEBUG [RS_CLOSE_REGION-regionserver/7b96b36745c6:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 49a51e328dbf56d3b52aa1d341d17e4e, disabling compactions & flushes 2024-11-21T19:49:02,163 DEBUG [RS:1;7b96b36745c6:45609 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-21T19:49:02,163 INFO [RS_CLOSE_REGION-regionserver/7b96b36745c6:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testBlockEvictionOnGracefulStop1732218540465,,1732218540469.49a51e328dbf56d3b52aa1d341d17e4e. 2024-11-21T19:49:02,163 DEBUG [RS:1;7b96b36745c6:45609 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T19:49:02,163 DEBUG [RS_CLOSE_REGION-regionserver/7b96b36745c6:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testBlockEvictionOnGracefulStop1732218540465,,1732218540469.49a51e328dbf56d3b52aa1d341d17e4e. 2024-11-21T19:49:02,163 DEBUG [RS_CLOSE_REGION-regionserver/7b96b36745c6:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testBlockEvictionOnGracefulStop1732218540465,,1732218540469.49a51e328dbf56d3b52aa1d341d17e4e. after waiting 0 ms 2024-11-21T19:49:02,163 INFO [RS:1;7b96b36745c6:45609 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-21T19:49:02,163 INFO [RS:1;7b96b36745c6:45609 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-21T19:49:02,163 DEBUG [RS_CLOSE_REGION-regionserver/7b96b36745c6:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testBlockEvictionOnGracefulStop1732218540465,,1732218540469.49a51e328dbf56d3b52aa1d341d17e4e. 2024-11-21T19:49:02,163 INFO [RS:1;7b96b36745c6:45609 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-21T19:49:02,163 INFO [RS:1;7b96b36745c6:45609 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-11-21T19:49:02,163 INFO [RS:1;7b96b36745c6:45609 {}] regionserver.HRegionServer(1321): Waiting on 2 regions to close 2024-11-21T19:49:02,164 DEBUG [RS:1;7b96b36745c6:45609 {}] regionserver.HRegionServer(1325): Online Regions={49a51e328dbf56d3b52aa1d341d17e4e=testBlockEvictionOnGracefulStop1732218540465,,1732218540469.49a51e328dbf56d3b52aa1d341d17e4e., 1588230740=hbase:meta,,1.1588230740} 2024-11-21T19:49:02,164 DEBUG [RS:1;7b96b36745c6:45609 {}] regionserver.HRegionServer(1351): Waiting on 1588230740, 49a51e328dbf56d3b52aa1d341d17e4e 2024-11-21T19:49:02,164 INFO [RS_CLOSE_META-regionserver/7b96b36745c6:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1662): Closing region hbase:meta,,1.1588230740 during a graceful stop, and cache persistence is on, so setting evict on close to false. 2024-11-21T19:49:02,164 DEBUG [RS_CLOSE_META-regionserver/7b96b36745c6:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-21T19:49:02,164 INFO [RS_CLOSE_META-regionserver/7b96b36745c6:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-21T19:49:02,164 DEBUG [RS_CLOSE_META-regionserver/7b96b36745c6:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-21T19:49:02,164 DEBUG [RS_CLOSE_META-regionserver/7b96b36745c6:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-21T19:49:02,164 DEBUG [RS_CLOSE_META-regionserver/7b96b36745c6:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-21T19:49:02,165 INFO [RS_CLOSE_META-regionserver/7b96b36745c6:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=1.75 KB heapSize=3.78 KB 2024-11-21T19:49:02,168 DEBUG [RS_CLOSE_REGION-regionserver/7b96b36745c6:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0b774921-3780-c936-4fe1-1d1dcb4ce861/545083c6-d774-d2ca-7409-0fbc2c22cf1a/data/default/testBlockEvictionOnGracefulStop1732218540465/49a51e328dbf56d3b52aa1d341d17e4e/recovered.edits/9.seqid, newMaxSeqId=9, maxSeqId=1 2024-11-21T19:49:02,169 INFO [RS_CLOSE_REGION-regionserver/7b96b36745c6:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testBlockEvictionOnGracefulStop1732218540465,,1732218540469.49a51e328dbf56d3b52aa1d341d17e4e. 2024-11-21T19:49:02,169 DEBUG [RS_CLOSE_REGION-regionserver/7b96b36745c6:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 49a51e328dbf56d3b52aa1d341d17e4e: Waiting for close lock at 1732218542163Running coprocessor pre-close hooks at 1732218542163Disabling compacts and flushes for region at 1732218542163Disabling writes for close at 1732218542163Writing region close event to WAL at 1732218542164 (+1 ms)Running coprocessor post-close hooks at 1732218542169 (+5 ms)Closed at 1732218542169 2024-11-21T19:49:02,169 DEBUG [RS_CLOSE_REGION-regionserver/7b96b36745c6:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testBlockEvictionOnGracefulStop1732218540465,,1732218540469.49a51e328dbf56d3b52aa1d341d17e4e. 2024-11-21T19:49:02,176 DEBUG [RS_CLOSE_META-regionserver/7b96b36745c6:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 52857c24a1964413a4d2a29dac0380c6 2024-11-21T19:49:02,191 DEBUG [RS_CLOSE_META-regionserver/7b96b36745c6:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0b774921-3780-c936-4fe1-1d1dcb4ce861/545083c6-d774-d2ca-7409-0fbc2c22cf1a/data/hbase/meta/1588230740/.tmp/info/52857c24a1964413a4d2a29dac0380c6 is 209, key is testBlockEvictionOnGracefulStop1732218540465,,1732218540469.49a51e328dbf56d3b52aa1d341d17e4e./info:regioninfo/1732218540832/Put/seqid=0 2024-11-21T19:49:02,191 INFO [RS_CLOSE_META-regionserver/7b96b36745c6:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.53 KB at sequenceid=11 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0b774921-3780-c936-4fe1-1d1dcb4ce861/545083c6-d774-d2ca-7409-0fbc2c22cf1a/data/hbase/meta/1588230740/.tmp/info/52857c24a1964413a4d2a29dac0380c6 2024-11-21T19:49:02,199 DEBUG [RS_CLOSE_META-regionserver/7b96b36745c6:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFile$WriterFactory(308): Unable to set drop behind on b1cb9e8185164438bba7e9d2c46d5af4 2024-11-21T19:49:02,212 DEBUG [RS_CLOSE_META-regionserver/7b96b36745c6:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0b774921-3780-c936-4fe1-1d1dcb4ce861/545083c6-d774-d2ca-7409-0fbc2c22cf1a/data/hbase/meta/1588230740/.tmp/ns/b1cb9e8185164438bba7e9d2c46d5af4 is 43, key is default/ns:d/1732218540380/Put/seqid=0 2024-11-21T19:49:02,213 INFO [RS_CLOSE_META-regionserver/7b96b36745c6:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=74 B at sequenceid=11 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0b774921-3780-c936-4fe1-1d1dcb4ce861/545083c6-d774-d2ca-7409-0fbc2c22cf1a/data/hbase/meta/1588230740/.tmp/ns/b1cb9e8185164438bba7e9d2c46d5af4 2024-11-21T19:49:02,217 INFO [regionserver/7b96b36745c6:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-21T19:49:02,221 DEBUG [RS_CLOSE_META-regionserver/7b96b36745c6:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFile$WriterFactory(308): Unable to set drop behind on bac66e20f4f54369b69f7de0858f7356 2024-11-21T19:49:02,233 DEBUG [RS_CLOSE_META-regionserver/7b96b36745c6:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0b774921-3780-c936-4fe1-1d1dcb4ce861/545083c6-d774-d2ca-7409-0fbc2c22cf1a/data/hbase/meta/1588230740/.tmp/table/bac66e20f4f54369b69f7de0858f7356 is 80, key is testBlockEvictionOnGracefulStop1732218540465/table:state/1732218540841/Put/seqid=0 2024-11-21T19:49:02,233 INFO [RS_CLOSE_META-regionserver/7b96b36745c6:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=152 B at sequenceid=11 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0b774921-3780-c936-4fe1-1d1dcb4ce861/545083c6-d774-d2ca-7409-0fbc2c22cf1a/data/hbase/meta/1588230740/.tmp/table/bac66e20f4f54369b69f7de0858f7356 2024-11-21T19:49:02,236 DEBUG [RS_CLOSE_META-regionserver/7b96b36745c6:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0b774921-3780-c936-4fe1-1d1dcb4ce861/545083c6-d774-d2ca-7409-0fbc2c22cf1a/data/hbase/meta/1588230740/.tmp/info/52857c24a1964413a4d2a29dac0380c6 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0b774921-3780-c936-4fe1-1d1dcb4ce861/545083c6-d774-d2ca-7409-0fbc2c22cf1a/data/hbase/meta/1588230740/info/52857c24a1964413a4d2a29dac0380c6 2024-11-21T19:49:02,237 DEBUG [RS_CLOSE_META-regionserver/7b96b36745c6:0-0 {event_type=M_RS_CLOSE_META}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0b774921-3780-c936-4fe1-1d1dcb4ce861/545083c6-d774-d2ca-7409-0fbc2c22cf1a/data/hbase/meta/1588230740/info/52857c24a1964413a4d2a29dac0380c6, delay=900 ms 2024-11-21T19:49:02,238 INFO [RS_CLOSE_META-regionserver/7b96b36745c6:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0b774921-3780-c936-4fe1-1d1dcb4ce861/545083c6-d774-d2ca-7409-0fbc2c22cf1a/data/hbase/meta/1588230740/info/52857c24a1964413a4d2a29dac0380c6, entries=10, sequenceid=11, filesize=7.0 K 2024-11-21T19:49:02,238 DEBUG [RS_CLOSE_META-regionserver/7b96b36745c6:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0b774921-3780-c936-4fe1-1d1dcb4ce861/545083c6-d774-d2ca-7409-0fbc2c22cf1a/data/hbase/meta/1588230740/.tmp/ns/b1cb9e8185164438bba7e9d2c46d5af4 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0b774921-3780-c936-4fe1-1d1dcb4ce861/545083c6-d774-d2ca-7409-0fbc2c22cf1a/data/hbase/meta/1588230740/ns/b1cb9e8185164438bba7e9d2c46d5af4 2024-11-21T19:49:02,239 DEBUG [RS_CLOSE_META-regionserver/7b96b36745c6:0-0 {event_type=M_RS_CLOSE_META}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0b774921-3780-c936-4fe1-1d1dcb4ce861/545083c6-d774-d2ca-7409-0fbc2c22cf1a/data/hbase/meta/1588230740/ns/b1cb9e8185164438bba7e9d2c46d5af4, delay=973 ms 2024-11-21T19:49:02,240 INFO [RS_CLOSE_META-regionserver/7b96b36745c6:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0b774921-3780-c936-4fe1-1d1dcb4ce861/545083c6-d774-d2ca-7409-0fbc2c22cf1a/data/hbase/meta/1588230740/ns/b1cb9e8185164438bba7e9d2c46d5af4, entries=2, sequenceid=11, filesize=5.0 K 2024-11-21T19:49:02,241 DEBUG [RS_CLOSE_META-regionserver/7b96b36745c6:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0b774921-3780-c936-4fe1-1d1dcb4ce861/545083c6-d774-d2ca-7409-0fbc2c22cf1a/data/hbase/meta/1588230740/.tmp/table/bac66e20f4f54369b69f7de0858f7356 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0b774921-3780-c936-4fe1-1d1dcb4ce861/545083c6-d774-d2ca-7409-0fbc2c22cf1a/data/hbase/meta/1588230740/table/bac66e20f4f54369b69f7de0858f7356 2024-11-21T19:49:02,242 DEBUG [RS_CLOSE_META-regionserver/7b96b36745c6:0-0 {event_type=M_RS_CLOSE_META}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0b774921-3780-c936-4fe1-1d1dcb4ce861/545083c6-d774-d2ca-7409-0fbc2c22cf1a/data/hbase/meta/1588230740/table/bac66e20f4f54369b69f7de0858f7356, delay=921 ms 2024-11-21T19:49:02,242 INFO [RS_CLOSE_META-regionserver/7b96b36745c6:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0b774921-3780-c936-4fe1-1d1dcb4ce861/545083c6-d774-d2ca-7409-0fbc2c22cf1a/data/hbase/meta/1588230740/table/bac66e20f4f54369b69f7de0858f7356, entries=2, sequenceid=11, filesize=5.3 K 2024-11-21T19:49:02,243 INFO [RS_CLOSE_META-regionserver/7b96b36745c6:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~1.75 KB/1797, heapSize ~3.48 KB/3568, currentSize=0 B/0 for 1588230740 in 79ms, sequenceid=11, compaction requested=false 2024-11-21T19:49:02,244 DEBUG [StoreFileCloser-1588230740-info-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0b774921-3780-c936-4fe1-1d1dcb4ce861/545083c6-d774-d2ca-7409-0fbc2c22cf1a/data/hbase/meta/1588230740/info/52857c24a1964413a4d2a29dac0380c6 2024-11-21T19:49:02,244 DEBUG [StoreFileCloser-1588230740-ns-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0b774921-3780-c936-4fe1-1d1dcb4ce861/545083c6-d774-d2ca-7409-0fbc2c22cf1a/data/hbase/meta/1588230740/ns/b1cb9e8185164438bba7e9d2c46d5af4 2024-11-21T19:49:02,245 DEBUG [StoreFileCloser-1588230740-table-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0b774921-3780-c936-4fe1-1d1dcb4ce861/545083c6-d774-d2ca-7409-0fbc2c22cf1a/data/hbase/meta/1588230740/table/bac66e20f4f54369b69f7de0858f7356 2024-11-21T19:49:02,249 DEBUG [RS_CLOSE_META-regionserver/7b96b36745c6:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0b774921-3780-c936-4fe1-1d1dcb4ce861/545083c6-d774-d2ca-7409-0fbc2c22cf1a/data/hbase/meta/1588230740/recovered.edits/14.seqid, newMaxSeqId=14, maxSeqId=1 2024-11-21T19:49:02,249 DEBUG [RS_CLOSE_META-regionserver/7b96b36745c6:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-21T19:49:02,249 INFO [RS_CLOSE_META-regionserver/7b96b36745c6:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-21T19:49:02,250 DEBUG [RS_CLOSE_META-regionserver/7b96b36745c6:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1732218542164Running coprocessor pre-close hooks at 1732218542164Disabling compacts and flushes for region at 1732218542164Disabling writes for close at 1732218542164Obtaining lock to block concurrent updates at 1732218542165 (+1 ms)Preparing flush snapshotting stores in 1588230740 at 1732218542165Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=1797, getHeapSize=3808, getOffHeapSize=0, getCellsCount=14 at 1732218542165Flushing stores of hbase:meta,,1.1588230740 at 1732218542166 (+1 ms)Flushing 1588230740/info: creating writer at 1732218542166Flushing 1588230740/info: appending metadata at 1732218542190 (+24 ms)Flushing 1588230740/info: closing flushed file at 1732218542190Flushing 1588230740/ns: creating writer at 1732218542193 (+3 ms)Flushing 1588230740/ns: appending metadata at 1732218542212 (+19 ms)Flushing 1588230740/ns: closing flushed file at 1732218542212Flushing 1588230740/table: creating writer at 1732218542215 (+3 ms)Flushing 1588230740/table: appending metadata at 1732218542232 (+17 ms)Flushing 1588230740/table: closing flushed file at 1732218542232Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@12d875c9: reopening flushed file at 1732218542236 (+4 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@101e8221: reopening flushed file at 1732218542238 (+2 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@3106699c: reopening flushed file at 1732218542240 (+2 ms)Finished flush of dataSize ~1.75 KB/1797, heapSize ~3.48 KB/3568, currentSize=0 B/0 for 1588230740 in 79ms, sequenceid=11, compaction requested=false at 1732218542243 (+3 ms)Writing region close event to WAL at 1732218542245 (+2 ms)Running coprocessor post-close hooks at 1732218542249 (+4 ms)Closed at 1732218542249 2024-11-21T19:49:02,250 DEBUG [RS_CLOSE_META-regionserver/7b96b36745c6:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-11-21T19:49:02,364 INFO [RS:1;7b96b36745c6:45609 {}] regionserver.HRegionServer(976): stopping server 7b96b36745c6,45609,1732218539412; all regions closed. 2024-11-21T19:49:02,366 DEBUG [RS:1;7b96b36745c6:45609 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0b774921-3780-c936-4fe1-1d1dcb4ce861/545083c6-d774-d2ca-7409-0fbc2c22cf1a/oldWALs 2024-11-21T19:49:02,366 INFO [RS:1;7b96b36745c6:45609 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 7b96b36745c6%2C45609%2C1732218539412.meta:.meta(num 1732218540325) 2024-11-21T19:49:02,367 DEBUG [RS:1;7b96b36745c6:45609 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0b774921-3780-c936-4fe1-1d1dcb4ce861/545083c6-d774-d2ca-7409-0fbc2c22cf1a/oldWALs 2024-11-21T19:49:02,367 INFO [RS:1;7b96b36745c6:45609 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 7b96b36745c6%2C45609%2C1732218539412:(num 1732218539957) 2024-11-21T19:49:02,367 DEBUG [RS:1;7b96b36745c6:45609 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T19:49:02,367 INFO [RS:1;7b96b36745c6:45609 {}] regionserver.LeaseManager(133): Closed leases 2024-11-21T19:49:02,368 INFO [RS:1;7b96b36745c6:45609 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-21T19:49:02,368 INFO [RS:1;7b96b36745c6:45609 {}] hbase.ChoreService(370): Chore service for: regionserver/7b96b36745c6:0 had [ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS] on shutdown 2024-11-21T19:49:02,368 INFO [RS:1;7b96b36745c6:45609 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-21T19:49:02,368 INFO [regionserver/7b96b36745c6:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-21T19:49:02,368 INFO [RS:1;7b96b36745c6:45609 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:45609 2024-11-21T19:49:02,372 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45609-0x10116d8419f0002, quorum=127.0.0.1:54341, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/7b96b36745c6,45609,1732218539412 2024-11-21T19:49:02,372 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44549-0x10116d8419f0000, quorum=127.0.0.1:54341, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-21T19:49:02,372 INFO [RS:1;7b96b36745c6:45609 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-21T19:49:02,374 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [7b96b36745c6,45609,1732218539412] 2024-11-21T19:49:02,376 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/7b96b36745c6,45609,1732218539412 already deleted, retry=false 2024-11-21T19:49:02,376 INFO [RegionServerTracker-0 {}] master.ServerManager(695): Processing expiration of 7b96b36745c6,45609,1732218539412 on 7b96b36745c6,44549,1732218539289 2024-11-21T19:49:02,380 DEBUG [RegionServerTracker-0 {}] procedure2.ProcedureExecutor(1139): Stored pid=7, state=RUNNABLE:SERVER_CRASH_START, hasLock=false; ServerCrashProcedure 7b96b36745c6,45609,1732218539412, splitWal=true, meta=true 2024-11-21T19:49:02,383 INFO [PEWorker-2 {}] procedure.ServerCrashProcedure(169): Start pid=7, state=RUNNABLE:SERVER_CRASH_START, hasLock=true; ServerCrashProcedure 7b96b36745c6,45609,1732218539412, splitWal=true, meta=true 2024-11-21T19:49:02,383 INFO [RegionServerTracker-0 {}] assignment.AssignmentManager(1999): Scheduled ServerCrashProcedure pid=7 for 7b96b36745c6,45609,1732218539412 (carryingMeta=true) 7b96b36745c6,45609,1732218539412/CRASHED/regionCount=2/lock=java.util.concurrent.locks.ReentrantReadWriteLock@6ddbf124[Write locks = 1, Read locks = 0], oldState=ONLINE. 2024-11-21T19:49:02,383 INFO [PEWorker-2 {}] procedure.ServerCrashProcedure(339): Splitting WALs pid=7, state=RUNNABLE:SERVER_CRASH_SPLIT_META_LOGS, hasLock=true; ServerCrashProcedure 7b96b36745c6,45609,1732218539412, splitWal=true, meta=true, isMeta: true 2024-11-21T19:49:02,384 DEBUG [PEWorker-2 {}] master.MasterWalManager(329): Renamed region directory: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0b774921-3780-c936-4fe1-1d1dcb4ce861/545083c6-d774-d2ca-7409-0fbc2c22cf1a/WALs/7b96b36745c6,45609,1732218539412-splitting 2024-11-21T19:49:02,384 INFO [PEWorker-2 {}] master.SplitLogManager(171): file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0b774921-3780-c936-4fe1-1d1dcb4ce861/545083c6-d774-d2ca-7409-0fbc2c22cf1a/WALs/7b96b36745c6,45609,1732218539412-splitting dir is empty, no logs to split. 2024-11-21T19:49:02,384 INFO [PEWorker-2 {}] master.SplitWALManager(105): 7b96b36745c6,45609,1732218539412 WAL count=0, meta=true 2024-11-21T19:49:02,386 INFO [PEWorker-2 {}] master.SplitLogManager(171): file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0b774921-3780-c936-4fe1-1d1dcb4ce861/545083c6-d774-d2ca-7409-0fbc2c22cf1a/WALs/7b96b36745c6,45609,1732218539412-splitting dir is empty, no logs to split. 2024-11-21T19:49:02,386 INFO [PEWorker-2 {}] master.SplitWALManager(105): 7b96b36745c6,45609,1732218539412 WAL count=0, meta=true 2024-11-21T19:49:02,386 DEBUG [PEWorker-2 {}] procedure.ServerCrashProcedure(329): Check if 7b96b36745c6,45609,1732218539412 WAL splitting is done? wals=0, meta=true 2024-11-21T19:49:02,387 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=8, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-11-21T19:49:02,388 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=8, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-21T19:49:02,389 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(269): Starting pid=8, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-11-21T19:49:02,474 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45609-0x10116d8419f0002, quorum=127.0.0.1:54341, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-21T19:49:02,474 INFO [RS:1;7b96b36745c6:45609 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-21T19:49:02,474 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45609-0x10116d8419f0002, quorum=127.0.0.1:54341, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-21T19:49:02,474 INFO [RS:1;7b96b36745c6:45609 {}] regionserver.HRegionServer(1031): Exiting; stopping=7b96b36745c6,45609,1732218539412; zookeeper connection closed. 2024-11-21T19:49:02,475 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@523efb9c {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@523efb9c 2024-11-21T19:49:02,539 DEBUG [7b96b36745c6:44549 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=1, allServersCount=1 2024-11-21T19:49:02,540 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=8 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=7b96b36745c6,43071,1732218539369 2024-11-21T19:49:02,541 INFO [PEWorker-4 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 7b96b36745c6,43071,1732218539369, state=OPENING 2024-11-21T19:49:02,543 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43071-0x10116d8419f0001, quorum=127.0.0.1:54341, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-21T19:49:02,543 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44549-0x10116d8419f0000, quorum=127.0.0.1:54341, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-21T19:49:02,543 DEBUG [PEWorker-4 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=8, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-21T19:49:02,543 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-21T19:49:02,543 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-21T19:49:02,543 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=9, ppid=8, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=7b96b36745c6,43071,1732218539369}] 2024-11-21T19:49:02,660 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/7b96b36745c6:0 server-side Connection retries=45 2024-11-21T19:49:02,660 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-21T19:49:02,660 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-21T19:49:02,660 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-21T19:49:02,660 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-21T19:49:02,660 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-21T19:49:02,660 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-21T19:49:02,660 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-21T19:49:02,661 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:44887 2024-11-21T19:49:02,662 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:44887 connecting to ZooKeeper ensemble=127.0.0.1:54341 2024-11-21T19:49:02,662 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-21T19:49:02,663 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-21T19:49:02,666 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:448870x0, quorum=127.0.0.1:54341, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-21T19:49:02,666 DEBUG [Time-limited test {}] zookeeper.ZKUtil(111): regionserver:448870x0, quorum=127.0.0.1:54341, baseZNode=/hbase Set watcher on existing znode=/hbase/running 2024-11-21T19:49:02,666 DEBUG [pool-298-thread-1 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: INIT 2024-11-21T19:49:02,666 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:44887-0x10116d8419f0003 connected 2024-11-21T19:49:02,667 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-21T19:49:02,668 INFO [Time-limited test {}] bucket.FileIOEngine(99): Allocating cache 400 MB, on the path:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/11a8a75e-d82a-1368-d32b-668baa66865a/bucket.cache 2024-11-21T19:49:02,668 INFO [Time-limited test {}] bucket.BucketCache(364): Instantiating BucketCache with acceptableFactor: 0.95, minFactor: 0.85, extraFreeFactor: 0.1, singleFactor: 0.25, multiFactor: 0.5, memoryFactor: 0.25, useStrongRef: false 2024-11-21T19:49:02,668 INFO [Time-limited test {}] bucket.BucketCache(478): Starting BucketCachePersisterThread 2024-11-21T19:49:02,668 INFO [Time-limited test {}] bucket.BucketCachePersister(38): BucketCachePersister started with interval: 100 2024-11-21T19:49:02,668 INFO [Thread-206 {}] bucket.BucketCache(1530): Started retrieving bucket cache from file 2024-11-21T19:49:02,668 INFO [Time-limited test {}] bucket.BucketCache(403): Started bucket cache; ioengine=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/11a8a75e-d82a-1368-d32b-668baa66865a/bucket.cache, capacity=400 MB, blockSize=64 KB, writerThreadNum=3, writerQLen=64, persistencePath=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/11a8a75e-d82a-1368-d32b-668baa66865a/bucket.persistence, bucketAllocator=org.apache.hadoop.hbase.io.hfile.bucket.BucketAllocator 2024-11-21T19:49:02,669 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-21T19:49:02,669 INFO [Thread-206 {}] bucket.BucketCache(1552): Reading new chunked format of persistence. 2024-11-21T19:49:02,669 DEBUG [Time-limited test {}] zookeeper.ZKUtil(111): regionserver:44887-0x10116d8419f0003, quorum=127.0.0.1:54341, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-11-21T19:49:02,671 INFO [Thread-206 {}] bucket.BucketCache(1705): Retrieved 1 of chunks with blockCount = 1. 2024-11-21T19:49:02,671 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:44887-0x10116d8419f0003, quorum=127.0.0.1:54341, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-21T19:49:02,672 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=44887 2024-11-21T19:49:02,673 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=44887 2024-11-21T19:49:02,676 DEBUG [Thread-206 {}] bucket.BucketCache(1569): Updating region size map after retrieving cached file list 2024-11-21T19:49:02,676 INFO [Thread-206 {}] bucket.BucketAllocator(343): Cache totalSize=418148352, buckets=199, bucket capacity=2101248=(4*525312)=(FEWEST_ITEMS_IN_BUCKET*(largest configured bucketcache size)) 2024-11-21T19:49:02,677 INFO [Thread-206 {}] bucket.BucketCache(1562): Bucket cache retrieved from file successfully with size: 1 2024-11-21T19:49:02,677 INFO [Thread-206 {}] bucket.BucketCache(413): Persistent bucket cache recovery from /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/11a8a75e-d82a-1368-d32b-668baa66865a/bucket.persistence is complete. 2024-11-21T19:49:02,680 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=44887 2024-11-21T19:49:02,681 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=44887 2024-11-21T19:49:02,684 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=44887 2024-11-21T19:49:02,685 INFO [RS:2;7b96b36745c6:44887 {}] regionserver.HRegionServer(746): ClusterId : f26c0bbc-ffc3-4204-8c91-2c4d735cdc03 2024-11-21T19:49:02,685 DEBUG [RS:2;7b96b36745c6:44887 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-21T19:49:02,688 DEBUG [RS:2;7b96b36745c6:44887 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-21T19:49:02,688 DEBUG [RS:2;7b96b36745c6:44887 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-21T19:49:02,690 DEBUG [RS:2;7b96b36745c6:44887 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-21T19:49:02,690 DEBUG [RS:2;7b96b36745c6:44887 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3170376d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=7b96b36745c6/172.17.0.2:0 2024-11-21T19:49:02,695 DEBUG [RSProcedureDispatcher-pool-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-21T19:49:02,697 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-7-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47935, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-21T19:49:02,699 INFO [RS_OPEN_META-regionserver/7b96b36745c6:0-0 {event_type=M_RS_OPEN_META, pid=9}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-11-21T19:49:02,699 INFO [RS_OPEN_META-regionserver/7b96b36745c6:0-0 {event_type=M_RS_OPEN_META, pid=9}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-21T19:49:02,700 INFO [RS_OPEN_META-regionserver/7b96b36745c6:0-0 {event_type=M_RS_OPEN_META, pid=9}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-11-21T19:49:02,701 INFO [RS_OPEN_META-regionserver/7b96b36745c6:0-0 {event_type=M_RS_OPEN_META, pid=9}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=7b96b36745c6%2C43071%2C1732218539369.meta, suffix=.meta, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0b774921-3780-c936-4fe1-1d1dcb4ce861/545083c6-d774-d2ca-7409-0fbc2c22cf1a/WALs/7b96b36745c6,43071,1732218539369, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0b774921-3780-c936-4fe1-1d1dcb4ce861/545083c6-d774-d2ca-7409-0fbc2c22cf1a/oldWALs, maxLogs=55 2024-11-21T19:49:02,704 DEBUG [RS:2;7b96b36745c6:44887 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:2;7b96b36745c6:44887 2024-11-21T19:49:02,704 INFO [RS:2;7b96b36745c6:44887 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-21T19:49:02,704 INFO [RS:2;7b96b36745c6:44887 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-21T19:49:02,704 DEBUG [RS:2;7b96b36745c6:44887 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-21T19:49:02,705 INFO [RS:2;7b96b36745c6:44887 {}] regionserver.HRegionServer(2659): reportForDuty to master=7b96b36745c6,44549,1732218539289 with port=44887, startcode=1732218542659 2024-11-21T19:49:02,705 DEBUG [RS:2;7b96b36745c6:44887 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-21T19:49:02,707 INFO [HMaster-EventLoopGroup-6-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44457, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=RegionServerStatusService 2024-11-21T19:49:02,708 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=44549 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 7b96b36745c6,44887,1732218542659 2024-11-21T19:49:02,708 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=44549 {}] master.ServerManager(517): Registering regionserver=7b96b36745c6,44887,1732218542659 2024-11-21T19:49:02,709 DEBUG [RS:2;7b96b36745c6:44887 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0b774921-3780-c936-4fe1-1d1dcb4ce861/545083c6-d774-d2ca-7409-0fbc2c22cf1a 2024-11-21T19:49:02,709 DEBUG [RS:2;7b96b36745c6:44887 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=file:/ 2024-11-21T19:49:02,709 DEBUG [RS:2;7b96b36745c6:44887 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-21T19:49:02,711 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44549-0x10116d8419f0000, quorum=127.0.0.1:54341, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-21T19:49:02,712 DEBUG [RS:2;7b96b36745c6:44887 {}] zookeeper.ZKUtil(111): regionserver:44887-0x10116d8419f0003, quorum=127.0.0.1:54341, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/7b96b36745c6,44887,1732218542659 2024-11-21T19:49:02,712 WARN [RS:2;7b96b36745c6:44887 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-21T19:49:02,712 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [7b96b36745c6,44887,1732218542659] 2024-11-21T19:49:02,712 INFO [RS:2;7b96b36745c6:44887 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-21T19:49:02,712 DEBUG [RS:2;7b96b36745c6:44887 {}] regionserver.HRegionServer(1793): logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0b774921-3780-c936-4fe1-1d1dcb4ce861/545083c6-d774-d2ca-7409-0fbc2c22cf1a/WALs/7b96b36745c6,44887,1732218542659 2024-11-21T19:49:02,718 INFO [RS:2;7b96b36745c6:44887 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-21T19:49:02,721 INFO [RS:2;7b96b36745c6:44887 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-21T19:49:02,721 INFO [RS_OPEN_META-regionserver/7b96b36745c6:0-0 {event_type=M_RS_OPEN_META, pid=9}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0b774921-3780-c936-4fe1-1d1dcb4ce861/545083c6-d774-d2ca-7409-0fbc2c22cf1a/WALs/7b96b36745c6,43071,1732218539369/7b96b36745c6%2C43071%2C1732218539369.meta.1732218542701.meta 2024-11-21T19:49:02,722 DEBUG [RS_OPEN_META-regionserver/7b96b36745c6:0-0 {event_type=M_RS_OPEN_META, pid=9}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-11-21T19:49:02,722 INFO [RS:2;7b96b36745c6:44887 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-21T19:49:02,722 INFO [RS:2;7b96b36745c6:44887 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-21T19:49:02,722 DEBUG [RS_OPEN_META-regionserver/7b96b36745c6:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-11-21T19:49:02,722 INFO [RS:2;7b96b36745c6:44887 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-21T19:49:02,722 DEBUG [RS_OPEN_META-regionserver/7b96b36745c6:0-0 {event_type=M_RS_OPEN_META, pid=9}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-11-21T19:49:02,722 DEBUG [RS_OPEN_META-regionserver/7b96b36745c6:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-11-21T19:49:02,722 INFO [RS_OPEN_META-regionserver/7b96b36745c6:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-11-21T19:49:02,722 INFO [RS:2;7b96b36745c6:44887 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-21T19:49:02,722 DEBUG [RS_OPEN_META-regionserver/7b96b36745c6:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-11-21T19:49:02,723 DEBUG [RS_OPEN_META-regionserver/7b96b36745c6:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-21T19:49:02,723 INFO [RS:2;7b96b36745c6:44887 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-21T19:49:02,723 DEBUG [RS_OPEN_META-regionserver/7b96b36745c6:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-11-21T19:49:02,723 DEBUG [RS_OPEN_META-regionserver/7b96b36745c6:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-11-21T19:49:02,723 DEBUG [RS:2;7b96b36745c6:44887 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/7b96b36745c6:0, corePoolSize=1, maxPoolSize=1 2024-11-21T19:49:02,723 DEBUG [RS:2;7b96b36745c6:44887 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/7b96b36745c6:0, corePoolSize=1, maxPoolSize=1 2024-11-21T19:49:02,723 DEBUG [RS:2;7b96b36745c6:44887 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/7b96b36745c6:0, corePoolSize=1, maxPoolSize=1 2024-11-21T19:49:02,723 DEBUG [RS:2;7b96b36745c6:44887 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/7b96b36745c6:0, corePoolSize=1, maxPoolSize=1 2024-11-21T19:49:02,723 DEBUG [RS:2;7b96b36745c6:44887 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/7b96b36745c6:0, corePoolSize=1, maxPoolSize=1 2024-11-21T19:49:02,723 DEBUG [RS:2;7b96b36745c6:44887 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/7b96b36745c6:0, corePoolSize=2, maxPoolSize=2 2024-11-21T19:49:02,723 DEBUG [RS:2;7b96b36745c6:44887 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/7b96b36745c6:0, corePoolSize=1, maxPoolSize=1 2024-11-21T19:49:02,723 DEBUG [RS:2;7b96b36745c6:44887 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/7b96b36745c6:0, corePoolSize=1, maxPoolSize=1 2024-11-21T19:49:02,723 DEBUG [RS:2;7b96b36745c6:44887 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/7b96b36745c6:0, corePoolSize=1, maxPoolSize=1 2024-11-21T19:49:02,723 DEBUG [RS:2;7b96b36745c6:44887 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/7b96b36745c6:0, corePoolSize=1, maxPoolSize=1 2024-11-21T19:49:02,723 DEBUG [RS:2;7b96b36745c6:44887 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/7b96b36745c6:0, corePoolSize=1, maxPoolSize=1 2024-11-21T19:49:02,724 DEBUG [RS:2;7b96b36745c6:44887 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/7b96b36745c6:0, corePoolSize=1, maxPoolSize=1 2024-11-21T19:49:02,724 DEBUG [RS:2;7b96b36745c6:44887 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/7b96b36745c6:0, corePoolSize=3, maxPoolSize=3 2024-11-21T19:49:02,724 DEBUG [RS:2;7b96b36745c6:44887 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/7b96b36745c6:0, corePoolSize=3, maxPoolSize=3 2024-11-21T19:49:02,724 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family info of region 1588230740 2024-11-21T19:49:02,725 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-21T19:49:02,725 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T19:49:02,725 INFO [RS:2;7b96b36745c6:44887 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-21T19:49:02,725 INFO [RS:2;7b96b36745c6:44887 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-21T19:49:02,725 INFO [RS:2;7b96b36745c6:44887 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-21T19:49:02,725 INFO [RS:2;7b96b36745c6:44887 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-21T19:49:02,725 INFO [RS:2;7b96b36745c6:44887 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-21T19:49:02,726 INFO [RS:2;7b96b36745c6:44887 {}] hbase.ChoreService(168): Chore ScheduledChore name=7b96b36745c6,44887,1732218542659-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-21T19:49:02,726 DEBUG [StoreFileOpener-1588230740-info-1 {}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0b774921-3780-c936-4fe1-1d1dcb4ce861/545083c6-d774-d2ca-7409-0fbc2c22cf1a/data/hbase/meta/1588230740/info/52857c24a1964413a4d2a29dac0380c6, delay=918 ms 2024-11-21T19:49:02,727 DEBUG [StoreOpener-1588230740-1 {}] regionserver.StoreEngine(278): loaded file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0b774921-3780-c936-4fe1-1d1dcb4ce861/545083c6-d774-d2ca-7409-0fbc2c22cf1a/data/hbase/meta/1588230740/info/52857c24a1964413a4d2a29dac0380c6 2024-11-21T19:49:02,727 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-21T19:49:02,727 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family ns of region 1588230740 2024-11-21T19:49:02,728 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-21T19:49:02,728 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T19:49:02,730 DEBUG [StoreFileOpener-1588230740-ns-1 {}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0b774921-3780-c936-4fe1-1d1dcb4ce861/545083c6-d774-d2ca-7409-0fbc2c22cf1a/data/hbase/meta/1588230740/ns/b1cb9e8185164438bba7e9d2c46d5af4, delay=938 ms 2024-11-21T19:49:02,731 DEBUG [StoreOpener-1588230740-1 {}] regionserver.StoreEngine(278): loaded file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0b774921-3780-c936-4fe1-1d1dcb4ce861/545083c6-d774-d2ca-7409-0fbc2c22cf1a/data/hbase/meta/1588230740/ns/b1cb9e8185164438bba7e9d2c46d5af4 2024-11-21T19:49:02,731 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-21T19:49:02,732 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family rep_barrier of region 1588230740 2024-11-21T19:49:02,732 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-21T19:49:02,732 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T19:49:02,733 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-21T19:49:02,733 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family table of region 1588230740 2024-11-21T19:49:02,733 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-21T19:49:02,733 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T19:49:02,736 DEBUG [StoreFileOpener-1588230740-table-1 {}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0b774921-3780-c936-4fe1-1d1dcb4ce861/545083c6-d774-d2ca-7409-0fbc2c22cf1a/data/hbase/meta/1588230740/table/bac66e20f4f54369b69f7de0858f7356, delay=920 ms 2024-11-21T19:49:02,737 DEBUG [StoreOpener-1588230740-1 {}] regionserver.StoreEngine(278): loaded file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0b774921-3780-c936-4fe1-1d1dcb4ce861/545083c6-d774-d2ca-7409-0fbc2c22cf1a/data/hbase/meta/1588230740/table/bac66e20f4f54369b69f7de0858f7356 2024-11-21T19:49:02,737 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-21T19:49:02,737 DEBUG [RS_OPEN_META-regionserver/7b96b36745c6:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-21T19:49:02,738 DEBUG [RS_OPEN_META-regionserver/7b96b36745c6:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0b774921-3780-c936-4fe1-1d1dcb4ce861/545083c6-d774-d2ca-7409-0fbc2c22cf1a/data/hbase/meta/1588230740 2024-11-21T19:49:02,738 DEBUG [RS_OPEN_META-regionserver/7b96b36745c6:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0b774921-3780-c936-4fe1-1d1dcb4ce861/545083c6-d774-d2ca-7409-0fbc2c22cf1a/data/hbase/meta/1588230740 2024-11-21T19:49:02,738 DEBUG [RS_OPEN_META-regionserver/7b96b36745c6:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-21T19:49:02,738 DEBUG [RS_OPEN_META-regionserver/7b96b36745c6:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-21T19:49:02,739 DEBUG [RS_OPEN_META-regionserver/7b96b36745c6:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-21T19:49:02,740 DEBUG [RS_OPEN_META-regionserver/7b96b36745c6:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-21T19:49:02,740 INFO [RS_OPEN_META-regionserver/7b96b36745c6:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=15; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=74969622, jitterRate=0.11713442206382751}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-21T19:49:02,740 DEBUG [RS_OPEN_META-regionserver/7b96b36745c6:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-11-21T19:49:02,741 DEBUG [RS_OPEN_META-regionserver/7b96b36745c6:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1732218542723Writing region info on filesystem at 1732218542723Initializing all the Stores at 1732218542723Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732218542723Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732218542724 (+1 ms)Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732218542724Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732218542724Cleaning up temporary data from old regions at 1732218542739 (+15 ms)Running coprocessor post-open hooks at 1732218542740 (+1 ms)Region opened successfully at 1732218542741 (+1 ms) 2024-11-21T19:49:02,742 INFO [RS_OPEN_META-regionserver/7b96b36745c6:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=9, masterSystemTime=1732218542695 2024-11-21T19:49:02,743 INFO [RS:2;7b96b36745c6:44887 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-21T19:49:02,744 INFO [RS:2;7b96b36745c6:44887 {}] hbase.ChoreService(168): Chore ScheduledChore name=7b96b36745c6,44887,1732218542659-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-21T19:49:02,744 INFO [RS:2;7b96b36745c6:44887 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-21T19:49:02,744 INFO [RS:2;7b96b36745c6:44887 {}] regionserver.Replication(171): 7b96b36745c6,44887,1732218542659 started 2024-11-21T19:49:02,744 DEBUG [RS_OPEN_META-regionserver/7b96b36745c6:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-11-21T19:49:02,744 INFO [RS_OPEN_META-regionserver/7b96b36745c6:0-0 {event_type=M_RS_OPEN_META, pid=9}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-11-21T19:49:02,745 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=8 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=15, regionLocation=7b96b36745c6,43071,1732218539369 2024-11-21T19:49:02,746 INFO [PEWorker-1 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 7b96b36745c6,43071,1732218539369, state=OPEN 2024-11-21T19:49:02,747 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44549-0x10116d8419f0000, quorum=127.0.0.1:54341, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-21T19:49:02,748 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43071-0x10116d8419f0001, quorum=127.0.0.1:54341, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-21T19:49:02,748 DEBUG [PEWorker-1 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=9, ppid=8, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=7b96b36745c6,43071,1732218539369 2024-11-21T19:49:02,748 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44887-0x10116d8419f0003, quorum=127.0.0.1:54341, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-21T19:49:02,748 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-21T19:49:02,748 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-21T19:49:02,748 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-21T19:49:02,750 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=9, resume processing ppid=8 2024-11-21T19:49:02,750 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=9, ppid=8, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=7b96b36745c6,43071,1732218539369 in 205 msec 2024-11-21T19:49:02,753 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=8, resume processing ppid=7 2024-11-21T19:49:02,753 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=8, ppid=7, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 363 msec 2024-11-21T19:49:02,753 INFO [PEWorker-3 {}] procedure.ServerCrashProcedure(207): 7b96b36745c6,45609,1732218539412 had 2 regions 2024-11-21T19:49:02,754 INFO [PEWorker-3 {}] procedure.ServerCrashProcedure(339): Splitting WALs pid=7, state=RUNNABLE:SERVER_CRASH_SPLIT_LOGS, hasLock=true; ServerCrashProcedure 7b96b36745c6,45609,1732218539412, splitWal=true, meta=true, isMeta: false 2024-11-21T19:49:02,755 INFO [PEWorker-3 {}] master.SplitLogManager(171): file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0b774921-3780-c936-4fe1-1d1dcb4ce861/545083c6-d774-d2ca-7409-0fbc2c22cf1a/WALs/7b96b36745c6,45609,1732218539412-splitting dir is empty, no logs to split. 2024-11-21T19:49:02,755 INFO [PEWorker-3 {}] master.SplitWALManager(105): 7b96b36745c6,45609,1732218539412 WAL count=0, meta=false 2024-11-21T19:49:02,756 INFO [PEWorker-3 {}] master.SplitLogManager(171): file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0b774921-3780-c936-4fe1-1d1dcb4ce861/545083c6-d774-d2ca-7409-0fbc2c22cf1a/WALs/7b96b36745c6,45609,1732218539412-splitting dir is empty, no logs to split. 2024-11-21T19:49:02,756 INFO [PEWorker-3 {}] master.SplitWALManager(105): 7b96b36745c6,45609,1732218539412 WAL count=0, meta=false 2024-11-21T19:49:02,756 DEBUG [PEWorker-3 {}] procedure.ServerCrashProcedure(329): Check if 7b96b36745c6,45609,1732218539412 WAL splitting is done? wals=0, meta=false 2024-11-21T19:49:02,758 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=10, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1732218540465, region=49a51e328dbf56d3b52aa1d341d17e4e, ASSIGN}] 2024-11-21T19:49:02,759 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=10, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1732218540465, region=49a51e328dbf56d3b52aa1d341d17e4e, ASSIGN 2024-11-21T19:49:02,760 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=10, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1732218540465, region=49a51e328dbf56d3b52aa1d341d17e4e, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-11-21T19:49:02,761 INFO [RS:2;7b96b36745c6:44887 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-21T19:49:02,761 INFO [RS:2;7b96b36745c6:44887 {}] regionserver.HRegionServer(1482): Serving as 7b96b36745c6,44887,1732218542659, RpcServer on 7b96b36745c6/172.17.0.2:44887, sessionid=0x10116d8419f0003 2024-11-21T19:49:02,761 DEBUG [RS:2;7b96b36745c6:44887 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-21T19:49:02,761 DEBUG [RS:2;7b96b36745c6:44887 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 7b96b36745c6,44887,1732218542659 2024-11-21T19:49:02,761 DEBUG [RS:2;7b96b36745c6:44887 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '7b96b36745c6,44887,1732218542659' 2024-11-21T19:49:02,761 DEBUG [RS:2;7b96b36745c6:44887 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-21T19:49:02,762 DEBUG [RS:2;7b96b36745c6:44887 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-21T19:49:02,762 DEBUG [RS:2;7b96b36745c6:44887 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-21T19:49:02,762 DEBUG [RS:2;7b96b36745c6:44887 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-21T19:49:02,762 DEBUG [RS:2;7b96b36745c6:44887 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 7b96b36745c6,44887,1732218542659 2024-11-21T19:49:02,762 DEBUG [RS:2;7b96b36745c6:44887 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '7b96b36745c6,44887,1732218542659' 2024-11-21T19:49:02,762 DEBUG [RS:2;7b96b36745c6:44887 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-21T19:49:02,763 DEBUG [RS:2;7b96b36745c6:44887 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-21T19:49:02,763 DEBUG [RS:2;7b96b36745c6:44887 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-21T19:49:02,763 INFO [RS:2;7b96b36745c6:44887 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-21T19:49:02,763 INFO [RS:2;7b96b36745c6:44887 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-21T19:49:02,864 INFO [RS:2;7b96b36745c6:44887 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-21T19:49:02,864 INFO [RS:2;7b96b36745c6:44887 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=7b96b36745c6%2C44887%2C1732218542659, suffix=, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0b774921-3780-c936-4fe1-1d1dcb4ce861/545083c6-d774-d2ca-7409-0fbc2c22cf1a/WALs/7b96b36745c6,44887,1732218542659, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0b774921-3780-c936-4fe1-1d1dcb4ce861/545083c6-d774-d2ca-7409-0fbc2c22cf1a/oldWALs, maxLogs=55 2024-11-21T19:49:02,886 INFO [RS:2;7b96b36745c6:44887 {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0b774921-3780-c936-4fe1-1d1dcb4ce861/545083c6-d774-d2ca-7409-0fbc2c22cf1a/WALs/7b96b36745c6,44887,1732218542659/7b96b36745c6%2C44887%2C1732218542659.1732218542865 2024-11-21T19:49:02,887 DEBUG [RS:2;7b96b36745c6:44887 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-11-21T19:49:02,910 DEBUG [7b96b36745c6:44549 {}] balancer.BalancerClusterState(204): Hosts are {7b96b36745c6=0} racks are {/default-rack=0} 2024-11-21T19:49:02,911 DEBUG [7b96b36745c6:44549 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-11-21T19:49:02,911 DEBUG [7b96b36745c6:44549 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-11-21T19:49:02,911 DEBUG [7b96b36745c6:44549 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-11-21T19:49:02,911 DEBUG [7b96b36745c6:44549 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-11-21T19:49:02,911 INFO [7b96b36745c6:44549 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-11-21T19:49:02,911 INFO [7b96b36745c6:44549 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-11-21T19:49:02,911 DEBUG [7b96b36745c6:44549 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-11-21T19:49:02,911 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=10 updating hbase:meta row=49a51e328dbf56d3b52aa1d341d17e4e, regionState=OPENING, regionLocation=7b96b36745c6,43071,1732218539369 2024-11-21T19:49:02,913 WARN [MiniHBaseClusterRegionServer-EventLoopGroup-8-3 {}] ipc.NettyRpcConnection$2(409): Exception encountered while connecting to the server 7b96b36745c6:45609 org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 7b96b36745c6/172.17.0.2:45609 Caused by: java.net.ConnectException: finishConnect(..) failed: Connection refused at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.newConnectException0(Errors.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.handleConnectErrno(Errors.java:131) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Socket.finishConnect(Socket.java:359) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.doFinishConnect(AbstractEpollChannel.java:715) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:692) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T19:49:02,914 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-8-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=7b96b36745c6,45609,1732218539412, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=7b96b36745c6,45609,1732218539412, seqNum=-1, error=java.net.ConnectException: Call to address=7b96b36745c6:45609 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 7b96b36745c6/172.17.0.2:45609 2024-11-21T19:49:02,915 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-8-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:meta,,1.1588230740, hostname=7b96b36745c6,45609,1732218539412, seqNum=-1 is java.net.ConnectException: finishConnect(..) failed: Connection refused 2024-11-21T19:49:02,915 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-8-3 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:meta,,1.1588230740, hostname=7b96b36745c6,45609,1732218539412, seqNum=-1 from cache 2024-11-21T19:49:02,916 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-8-3 {}] ipc.FailedServers(52): Added failed server with address 7b96b36745c6:45609 to list caused by org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 7b96b36745c6/172.17.0.2:45609 2024-11-21T19:49:03,026 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-21T19:49:03,026 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=7b96b36745c6,43071,1732218539369, seqNum=-1] 2024-11-21T19:49:03,027 DEBUG [Async-Client-Retry-Timer-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-21T19:49:03,029 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-7-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59611, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-21T19:49:03,032 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-8-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=10, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1732218540465, region=49a51e328dbf56d3b52aa1d341d17e4e, ASSIGN because future has completed 2024-11-21T19:49:03,032 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=11, ppid=10, state=RUNNABLE, hasLock=false; OpenRegionProcedure 49a51e328dbf56d3b52aa1d341d17e4e, server=7b96b36745c6,43071,1732218539369}] 2024-11-21T19:49:03,187 INFO [RS_OPEN_REGION-regionserver/7b96b36745c6:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] handler.AssignRegionHandler(132): Open testBlockEvictionOnGracefulStop1732218540465,,1732218540469.49a51e328dbf56d3b52aa1d341d17e4e. 2024-11-21T19:49:03,187 DEBUG [RS_OPEN_REGION-regionserver/7b96b36745c6:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(7752): Opening region: {ENCODED => 49a51e328dbf56d3b52aa1d341d17e4e, NAME => 'testBlockEvictionOnGracefulStop1732218540465,,1732218540469.49a51e328dbf56d3b52aa1d341d17e4e.', STARTKEY => '', ENDKEY => ''} 2024-11-21T19:49:03,188 DEBUG [RS_OPEN_REGION-regionserver/7b96b36745c6:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testBlockEvictionOnGracefulStop1732218540465 49a51e328dbf56d3b52aa1d341d17e4e 2024-11-21T19:49:03,188 DEBUG [RS_OPEN_REGION-regionserver/7b96b36745c6:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(898): Instantiated testBlockEvictionOnGracefulStop1732218540465,,1732218540469.49a51e328dbf56d3b52aa1d341d17e4e.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-21T19:49:03,188 DEBUG [RS_OPEN_REGION-regionserver/7b96b36745c6:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(7794): checking encryption for 49a51e328dbf56d3b52aa1d341d17e4e 2024-11-21T19:49:03,188 DEBUG [RS_OPEN_REGION-regionserver/7b96b36745c6:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(7797): checking classloading for 49a51e328dbf56d3b52aa1d341d17e4e 2024-11-21T19:49:03,189 INFO [StoreOpener-49a51e328dbf56d3b52aa1d341d17e4e-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family family of region 49a51e328dbf56d3b52aa1d341d17e4e 2024-11-21T19:49:03,190 INFO [StoreOpener-49a51e328dbf56d3b52aa1d341d17e4e-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 49a51e328dbf56d3b52aa1d341d17e4e columnFamilyName family 2024-11-21T19:49:03,190 DEBUG [StoreOpener-49a51e328dbf56d3b52aa1d341d17e4e-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T19:49:03,191 DEBUG [StoreFileOpener-49a51e328dbf56d3b52aa1d341d17e4e-family-1 {}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0b774921-3780-c936-4fe1-1d1dcb4ce861/545083c6-d774-d2ca-7409-0fbc2c22cf1a/data/default/testBlockEvictionOnGracefulStop1732218540465/49a51e328dbf56d3b52aa1d341d17e4e/family/da9e36673a60423a945995608d46d07f, delay=983 ms 2024-11-21T19:49:03,192 DEBUG [StoreOpener-49a51e328dbf56d3b52aa1d341d17e4e-1 {}] regionserver.StoreEngine(278): loaded file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0b774921-3780-c936-4fe1-1d1dcb4ce861/545083c6-d774-d2ca-7409-0fbc2c22cf1a/data/default/testBlockEvictionOnGracefulStop1732218540465/49a51e328dbf56d3b52aa1d341d17e4e/family/da9e36673a60423a945995608d46d07f 2024-11-21T19:49:03,193 INFO [StoreOpener-49a51e328dbf56d3b52aa1d341d17e4e-1 {}] regionserver.HStore(327): Store=49a51e328dbf56d3b52aa1d341d17e4e/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-21T19:49:03,193 DEBUG [RS_OPEN_REGION-regionserver/7b96b36745c6:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(1038): replaying wal for 49a51e328dbf56d3b52aa1d341d17e4e 2024-11-21T19:49:03,193 DEBUG [RS_OPEN_REGION-regionserver/7b96b36745c6:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0b774921-3780-c936-4fe1-1d1dcb4ce861/545083c6-d774-d2ca-7409-0fbc2c22cf1a/data/default/testBlockEvictionOnGracefulStop1732218540465/49a51e328dbf56d3b52aa1d341d17e4e 2024-11-21T19:49:03,194 DEBUG [RS_OPEN_REGION-regionserver/7b96b36745c6:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0b774921-3780-c936-4fe1-1d1dcb4ce861/545083c6-d774-d2ca-7409-0fbc2c22cf1a/data/default/testBlockEvictionOnGracefulStop1732218540465/49a51e328dbf56d3b52aa1d341d17e4e 2024-11-21T19:49:03,194 DEBUG [RS_OPEN_REGION-regionserver/7b96b36745c6:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(1048): stopping wal replay for 49a51e328dbf56d3b52aa1d341d17e4e 2024-11-21T19:49:03,194 DEBUG [RS_OPEN_REGION-regionserver/7b96b36745c6:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(1060): Cleaning up temporary data for 49a51e328dbf56d3b52aa1d341d17e4e 2024-11-21T19:49:03,195 DEBUG [RS_OPEN_REGION-regionserver/7b96b36745c6:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(1093): writing seq id for 49a51e328dbf56d3b52aa1d341d17e4e 2024-11-21T19:49:03,195 INFO [RS_OPEN_REGION-regionserver/7b96b36745c6:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(1114): Opened 49a51e328dbf56d3b52aa1d341d17e4e; next sequenceid=10; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=63119193, jitterRate=-0.05945073068141937}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-21T19:49:03,196 DEBUG [RS_OPEN_REGION-regionserver/7b96b36745c6:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 49a51e328dbf56d3b52aa1d341d17e4e 2024-11-21T19:49:03,197 DEBUG [RS_OPEN_REGION-regionserver/7b96b36745c6:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(1006): Region open journal for 49a51e328dbf56d3b52aa1d341d17e4e: Running coprocessor pre-open hook at 1732218543188Writing region info on filesystem at 1732218543188Initializing all the Stores at 1732218543188Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732218543188Cleaning up temporary data from old regions at 1732218543194 (+6 ms)Running coprocessor post-open hooks at 1732218543196 (+2 ms)Region opened successfully at 1732218543197 (+1 ms) 2024-11-21T19:49:03,197 INFO [RS_OPEN_REGION-regionserver/7b96b36745c6:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegionServer(2236): Post open deploy tasks for testBlockEvictionOnGracefulStop1732218540465,,1732218540469.49a51e328dbf56d3b52aa1d341d17e4e., pid=11, masterSystemTime=1732218543185 2024-11-21T19:49:03,199 DEBUG [RS_OPEN_REGION-regionserver/7b96b36745c6:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegionServer(2266): Finished post open deploy task for testBlockEvictionOnGracefulStop1732218540465,,1732218540469.49a51e328dbf56d3b52aa1d341d17e4e. 2024-11-21T19:49:03,200 INFO [RS_OPEN_REGION-regionserver/7b96b36745c6:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] handler.AssignRegionHandler(153): Opened testBlockEvictionOnGracefulStop1732218540465,,1732218540469.49a51e328dbf56d3b52aa1d341d17e4e. 2024-11-21T19:49:03,200 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=10 updating hbase:meta row=49a51e328dbf56d3b52aa1d341d17e4e, regionState=OPEN, openSeqNum=10, regionLocation=7b96b36745c6,43071,1732218539369 2024-11-21T19:49:03,202 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-8-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=11, ppid=10, state=RUNNABLE, hasLock=false; OpenRegionProcedure 49a51e328dbf56d3b52aa1d341d17e4e, server=7b96b36745c6,43071,1732218539369 because future has completed 2024-11-21T19:49:03,205 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=11, resume processing ppid=10 2024-11-21T19:49:03,205 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=11, ppid=10, state=SUCCESS, hasLock=false; OpenRegionProcedure 49a51e328dbf56d3b52aa1d341d17e4e, server=7b96b36745c6,43071,1732218539369 in 171 msec 2024-11-21T19:49:03,206 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=10, resume processing ppid=7 2024-11-21T19:49:03,206 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=10, ppid=7, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1732218540465, region=49a51e328dbf56d3b52aa1d341d17e4e, ASSIGN in 447 msec 2024-11-21T19:49:03,206 INFO [PEWorker-1 {}] procedure.ServerCrashProcedure(291): removed crashed server 7b96b36745c6,45609,1732218539412 after splitting done 2024-11-21T19:49:03,208 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=7, state=SUCCESS, hasLock=false; ServerCrashProcedure 7b96b36745c6,45609,1732218539412, splitWal=true, meta=true in 830 msec 2024-11-21T19:49:03,261 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster 2024-11-21T19:49:03,261 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-21T19:49:03,261 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79) at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611) at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.io.hfile.TestBlockEvictionOnRegionMovement.tearDown(TestBlockEvictionOnRegionMovement.java:174) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-21T19:49:03,262 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T19:49:03,262 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T19:49:03,262 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-21T19:49:03,262 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2024-11-21T19:49:03,262 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=672656311, stopped=false 2024-11-21T19:49:03,262 INFO [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=7b96b36745c6,44549,1732218539289 2024-11-21T19:49:03,264 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43071-0x10116d8419f0001, quorum=127.0.0.1:54341, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-21T19:49:03,264 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44887-0x10116d8419f0003, quorum=127.0.0.1:54341, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-21T19:49:03,264 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43071-0x10116d8419f0001, quorum=127.0.0.1:54341, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-21T19:49:03,264 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44549-0x10116d8419f0000, quorum=127.0.0.1:54341, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-21T19:49:03,264 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44887-0x10116d8419f0003, quorum=127.0.0.1:54341, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-21T19:49:03,264 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-21T19:49:03,264 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44549-0x10116d8419f0000, quorum=127.0.0.1:54341, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-21T19:49:03,265 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-21T19:49:03,266 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:43071-0x10116d8419f0001, quorum=127.0.0.1:54341, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-21T19:49:03,266 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:44887-0x10116d8419f0003, quorum=127.0.0.1:54341, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-21T19:49:03,267 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:44549-0x10116d8419f0000, quorum=127.0.0.1:54341, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-21T19:49:03,267 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277) at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265) at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416) at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.io.hfile.TestBlockEvictionOnRegionMovement.tearDown(TestBlockEvictionOnRegionMovement.java:174) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-21T19:49:03,267 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T19:49:03,268 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '7b96b36745c6,43071,1732218539369' ***** 2024-11-21T19:49:03,268 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-11-21T19:49:03,268 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '7b96b36745c6,44887,1732218542659' ***** 2024-11-21T19:49:03,268 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-11-21T19:49:03,268 INFO [RS:0;7b96b36745c6:43071 {}] bucket.BucketCache(1738): Disabling cache 2024-11-21T19:49:03,268 INFO [RS:0;7b96b36745c6:43071 {}] bucket.BucketCache(1762): Shutdown bucket cache: IO persistent=true; path to write=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/11a8a75e-d82a-1368-d32b-668baa66865a/bucket.persistence 2024-11-21T19:49:03,268 INFO [RS:2;7b96b36745c6:44887 {}] bucket.BucketCache(1738): Disabling cache 2024-11-21T19:49:03,268 INFO [Time-limited test-BucketCacheWriter-0 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-0 exiting, cacheEnabled=false 2024-11-21T19:49:03,268 INFO [Time-limited test-BucketCacheWriter-1 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-1 exiting, cacheEnabled=false 2024-11-21T19:49:03,268 INFO [Time-limited test-BucketCacheWriter-0 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-0 exiting, cacheEnabled=false 2024-11-21T19:49:03,268 INFO [Time-limited test-BucketCacheWriter-1 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-1 exiting, cacheEnabled=false 2024-11-21T19:49:03,268 INFO [RS:2;7b96b36745c6:44887 {}] bucket.BucketCache(1762): Shutdown bucket cache: IO persistent=true; path to write=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/11a8a75e-d82a-1368-d32b-668baa66865a/bucket.persistence 2024-11-21T19:49:03,269 INFO [Time-limited test-BucketCacheWriter-2 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-2 exiting, cacheEnabled=false 2024-11-21T19:49:03,269 INFO [RS:2;7b96b36745c6:44887 {}] bucket.BucketCache(1768): Shutting down cache persister thread. 2024-11-21T19:49:03,269 INFO [Time-limited test-BucketCacheWriter-2 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-2 exiting, cacheEnabled=false 2024-11-21T19:49:03,269 INFO [RS:0;7b96b36745c6:43071 {}] bucket.BucketCache(1768): Shutting down cache persister thread. 2024-11-21T19:49:03,270 WARN [bucket-cache-persister {}] bucket.BucketCachePersister(64): Interrupting BucketCachePersister thread. java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCachePersister.run(BucketCachePersister.java:45) ~[classes/:?] 2024-11-21T19:49:03,270 WARN [bucket-cache-persister {}] bucket.BucketCachePersister(64): Interrupting BucketCachePersister thread. java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCachePersister.run(BucketCachePersister.java:45) ~[classes/:?] 2024-11-21T19:49:03,279 DEBUG [RS:2;7b96b36745c6:44887 {}] bucket.BucketCache(1487): Thread RS:2;7b96b36745c6:44887 started persisting bucket cache to file 2024-11-21T19:49:03,280 DEBUG [RS:2;7b96b36745c6:44887 {}] bucket.BucketCache(1494): Persist in new chunked persistence format. 2024-11-21T19:49:03,280 DEBUG [RS:2;7b96b36745c6:44887 {}] bucket.BucketCache(1673): persistToFile: before persisting backing map size: 1, fullycachedFiles size: 0, chunkSize: 10000 2024-11-21T19:49:03,280 DEBUG [RS:0;7b96b36745c6:43071 {}] bucket.BucketCache(1487): Thread RS:0;7b96b36745c6:43071 started persisting bucket cache to file 2024-11-21T19:49:03,280 DEBUG [RS:0;7b96b36745c6:43071 {}] bucket.BucketCache(1494): Persist in new chunked persistence format. 2024-11-21T19:49:03,280 DEBUG [RS:0;7b96b36745c6:43071 {}] bucket.BucketCache(1673): persistToFile: before persisting backing map size: 0, fullycachedFiles size: 0, chunkSize: 10000 2024-11-21T19:49:03,283 DEBUG [RS:0;7b96b36745c6:43071 {}] bucket.BucketCache(1680): persistToFile: after persisting backing map size: 0, fullycachedFiles size: 0 2024-11-21T19:49:03,283 DEBUG [RS:0;7b96b36745c6:43071 {}] bucket.BucketCache(1498): PersistToFile: after persisting backing map size: 0, fullycachedFiles size: 0, file name: bucket.persistence1732218543280 2024-11-21T19:49:03,283 DEBUG [RS:0;7b96b36745c6:43071 {}] bucket.BucketCache(1509): Thread RS:0;7b96b36745c6:43071 finished persisting bucket cache to file, renaming 2024-11-21T19:49:03,283 DEBUG [RS:2;7b96b36745c6:44887 {}] bucket.BucketCache(1680): persistToFile: after persisting backing map size: 1, fullycachedFiles size: 0 2024-11-21T19:49:03,283 DEBUG [RS:2;7b96b36745c6:44887 {}] bucket.BucketCache(1498): PersistToFile: after persisting backing map size: 1, fullycachedFiles size: 0, file name: bucket.persistence1732218543279 2024-11-21T19:49:03,283 DEBUG [RS:2;7b96b36745c6:44887 {}] bucket.BucketCache(1509): Thread RS:2;7b96b36745c6:44887 finished persisting bucket cache to file, renaming 2024-11-21T19:49:03,283 INFO [RS:0;7b96b36745c6:43071 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-21T19:49:03,283 INFO [RS:0;7b96b36745c6:43071 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-21T19:49:03,283 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-21T19:49:03,283 INFO [RS:0;7b96b36745c6:43071 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-21T19:49:03,284 INFO [RS:0;7b96b36745c6:43071 {}] regionserver.HRegionServer(3091): Received CLOSE for 49a51e328dbf56d3b52aa1d341d17e4e 2024-11-21T19:49:03,284 INFO [RS:0;7b96b36745c6:43071 {}] regionserver.HRegionServer(959): stopping server 7b96b36745c6,43071,1732218539369 2024-11-21T19:49:03,284 INFO [RS:0;7b96b36745c6:43071 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-21T19:49:03,284 INFO [RS:0;7b96b36745c6:43071 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;7b96b36745c6:43071. 2024-11-21T19:49:03,284 DEBUG [RS:0;7b96b36745c6:43071 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-21T19:49:03,284 INFO [RS_CLOSE_REGION-regionserver/7b96b36745c6:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1662): Closing region testBlockEvictionOnGracefulStop1732218540465,,1732218540469.49a51e328dbf56d3b52aa1d341d17e4e. during a graceful stop, and cache persistence is on, so setting evict on close to false. 2024-11-21T19:49:03,284 DEBUG [RS:0;7b96b36745c6:43071 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T19:49:03,284 INFO [RS:2;7b96b36745c6:44887 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-21T19:49:03,284 DEBUG [RS_CLOSE_REGION-regionserver/7b96b36745c6:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 49a51e328dbf56d3b52aa1d341d17e4e, disabling compactions & flushes 2024-11-21T19:49:03,284 INFO [RS:0;7b96b36745c6:43071 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-21T19:49:03,284 INFO [RS:2;7b96b36745c6:44887 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-21T19:49:03,284 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-21T19:49:03,284 INFO [RS_CLOSE_REGION-regionserver/7b96b36745c6:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testBlockEvictionOnGracefulStop1732218540465,,1732218540469.49a51e328dbf56d3b52aa1d341d17e4e. 2024-11-21T19:49:03,284 INFO [RS:0;7b96b36745c6:43071 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-21T19:49:03,284 INFO [RS:2;7b96b36745c6:44887 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-21T19:49:03,284 INFO [RS:0;7b96b36745c6:43071 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-21T19:49:03,284 DEBUG [RS_CLOSE_REGION-regionserver/7b96b36745c6:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testBlockEvictionOnGracefulStop1732218540465,,1732218540469.49a51e328dbf56d3b52aa1d341d17e4e. 2024-11-21T19:49:03,284 INFO [RS:2;7b96b36745c6:44887 {}] regionserver.HRegionServer(959): stopping server 7b96b36745c6,44887,1732218542659 2024-11-21T19:49:03,284 INFO [RS:0;7b96b36745c6:43071 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-11-21T19:49:03,284 INFO [RS:2;7b96b36745c6:44887 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-21T19:49:03,284 DEBUG [RS_CLOSE_REGION-regionserver/7b96b36745c6:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testBlockEvictionOnGracefulStop1732218540465,,1732218540469.49a51e328dbf56d3b52aa1d341d17e4e. after waiting 0 ms 2024-11-21T19:49:03,285 DEBUG [RS_CLOSE_REGION-regionserver/7b96b36745c6:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testBlockEvictionOnGracefulStop1732218540465,,1732218540469.49a51e328dbf56d3b52aa1d341d17e4e. 2024-11-21T19:49:03,285 INFO [RS:2;7b96b36745c6:44887 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:2;7b96b36745c6:44887. 2024-11-21T19:49:03,285 INFO [RS:0;7b96b36745c6:43071 {}] regionserver.HRegionServer(1321): Waiting on 2 regions to close 2024-11-21T19:49:03,285 DEBUG [RS:2;7b96b36745c6:44887 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-21T19:49:03,285 DEBUG [RS:0;7b96b36745c6:43071 {}] regionserver.HRegionServer(1325): Online Regions={49a51e328dbf56d3b52aa1d341d17e4e=testBlockEvictionOnGracefulStop1732218540465,,1732218540469.49a51e328dbf56d3b52aa1d341d17e4e., 1588230740=hbase:meta,,1.1588230740} 2024-11-21T19:49:03,285 DEBUG [RS:2;7b96b36745c6:44887 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T19:49:03,285 DEBUG [RS:0;7b96b36745c6:43071 {}] regionserver.HRegionServer(1351): Waiting on 1588230740, 49a51e328dbf56d3b52aa1d341d17e4e 2024-11-21T19:49:03,285 INFO [RS_CLOSE_META-regionserver/7b96b36745c6:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1662): Closing region hbase:meta,,1.1588230740 during a graceful stop, and cache persistence is on, so setting evict on close to false. 2024-11-21T19:49:03,285 INFO [RS:2;7b96b36745c6:44887 {}] regionserver.HRegionServer(976): stopping server 7b96b36745c6,44887,1732218542659; all regions closed. 2024-11-21T19:49:03,285 DEBUG [RS_CLOSE_META-regionserver/7b96b36745c6:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-21T19:49:03,285 INFO [RS_CLOSE_META-regionserver/7b96b36745c6:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-21T19:49:03,285 DEBUG [RS_CLOSE_META-regionserver/7b96b36745c6:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-21T19:49:03,285 DEBUG [RS_CLOSE_META-regionserver/7b96b36745c6:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-21T19:49:03,285 DEBUG [RS_CLOSE_META-regionserver/7b96b36745c6:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-21T19:49:03,285 INFO [RS_CLOSE_META-regionserver/7b96b36745c6:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=1.21 KB heapSize=2.80 KB 2024-11-21T19:49:03,286 DEBUG [StoreFileCloser-49a51e328dbf56d3b52aa1d341d17e4e-family-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0b774921-3780-c936-4fe1-1d1dcb4ce861/545083c6-d774-d2ca-7409-0fbc2c22cf1a/data/default/testBlockEvictionOnGracefulStop1732218540465/49a51e328dbf56d3b52aa1d341d17e4e/family/da9e36673a60423a945995608d46d07f 2024-11-21T19:49:03,287 DEBUG [RS:2;7b96b36745c6:44887 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0b774921-3780-c936-4fe1-1d1dcb4ce861/545083c6-d774-d2ca-7409-0fbc2c22cf1a/oldWALs 2024-11-21T19:49:03,287 INFO [RS:2;7b96b36745c6:44887 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 7b96b36745c6%2C44887%2C1732218542659:(num 1732218542865) 2024-11-21T19:49:03,287 DEBUG [RS:2;7b96b36745c6:44887 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T19:49:03,287 INFO [RS:2;7b96b36745c6:44887 {}] regionserver.LeaseManager(133): Closed leases 2024-11-21T19:49:03,287 INFO [RS:2;7b96b36745c6:44887 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-21T19:49:03,287 INFO [RS:2;7b96b36745c6:44887 {}] hbase.ChoreService(370): Chore service for: regionserver/7b96b36745c6:0 had [ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS] on shutdown 2024-11-21T19:49:03,287 INFO [RS:2;7b96b36745c6:44887 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-21T19:49:03,288 INFO [regionserver/7b96b36745c6:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-21T19:49:03,288 INFO [RS:2;7b96b36745c6:44887 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-21T19:49:03,288 INFO [RS:2;7b96b36745c6:44887 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-21T19:49:03,288 INFO [RS:2;7b96b36745c6:44887 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-21T19:49:03,288 INFO [RS:2;7b96b36745c6:44887 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:44887 2024-11-21T19:49:03,290 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44887-0x10116d8419f0003, quorum=127.0.0.1:54341, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/7b96b36745c6,44887,1732218542659 2024-11-21T19:49:03,290 INFO [RS:2;7b96b36745c6:44887 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-21T19:49:03,290 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44549-0x10116d8419f0000, quorum=127.0.0.1:54341, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-21T19:49:03,291 DEBUG [RS_CLOSE_REGION-regionserver/7b96b36745c6:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0b774921-3780-c936-4fe1-1d1dcb4ce861/545083c6-d774-d2ca-7409-0fbc2c22cf1a/data/default/testBlockEvictionOnGracefulStop1732218540465/49a51e328dbf56d3b52aa1d341d17e4e/recovered.edits/12.seqid, newMaxSeqId=12, maxSeqId=9 2024-11-21T19:49:03,291 INFO [RS_CLOSE_REGION-regionserver/7b96b36745c6:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testBlockEvictionOnGracefulStop1732218540465,,1732218540469.49a51e328dbf56d3b52aa1d341d17e4e. 2024-11-21T19:49:03,291 DEBUG [RS_CLOSE_REGION-regionserver/7b96b36745c6:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 49a51e328dbf56d3b52aa1d341d17e4e: Waiting for close lock at 1732218543284Running coprocessor pre-close hooks at 1732218543284Disabling compacts and flushes for region at 1732218543284Disabling writes for close at 1732218543285 (+1 ms)Writing region close event to WAL at 1732218543286 (+1 ms)Running coprocessor post-close hooks at 1732218543291 (+5 ms)Closed at 1732218543291 2024-11-21T19:49:03,291 DEBUG [RS_CLOSE_REGION-regionserver/7b96b36745c6:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testBlockEvictionOnGracefulStop1732218540465,,1732218540469.49a51e328dbf56d3b52aa1d341d17e4e. 2024-11-21T19:49:03,292 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [7b96b36745c6,44887,1732218542659] 2024-11-21T19:49:03,294 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/7b96b36745c6,44887,1732218542659 already deleted, retry=false 2024-11-21T19:49:03,294 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; 7b96b36745c6,44887,1732218542659 expired; onlineServers=1 2024-11-21T19:49:03,297 DEBUG [RS_CLOSE_META-regionserver/7b96b36745c6:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 520f9ebe9ef94511be12a77464f18232 2024-11-21T19:49:03,311 DEBUG [RS_CLOSE_META-regionserver/7b96b36745c6:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0b774921-3780-c936-4fe1-1d1dcb4ce861/545083c6-d774-d2ca-7409-0fbc2c22cf1a/data/hbase/meta/1588230740/.tmp/info/520f9ebe9ef94511be12a77464f18232 is 209, key is testBlockEvictionOnGracefulStop1732218540465,,1732218540469.49a51e328dbf56d3b52aa1d341d17e4e./info:regioninfo/1732218543200/Put/seqid=0 2024-11-21T19:49:03,312 INFO [RS_CLOSE_META-regionserver/7b96b36745c6:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.21 KB at sequenceid=19 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0b774921-3780-c936-4fe1-1d1dcb4ce861/545083c6-d774-d2ca-7409-0fbc2c22cf1a/data/hbase/meta/1588230740/.tmp/info/520f9ebe9ef94511be12a77464f18232 2024-11-21T19:49:03,314 DEBUG [RS_CLOSE_META-regionserver/7b96b36745c6:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0b774921-3780-c936-4fe1-1d1dcb4ce861/545083c6-d774-d2ca-7409-0fbc2c22cf1a/data/hbase/meta/1588230740/.tmp/info/520f9ebe9ef94511be12a77464f18232 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0b774921-3780-c936-4fe1-1d1dcb4ce861/545083c6-d774-d2ca-7409-0fbc2c22cf1a/data/hbase/meta/1588230740/info/520f9ebe9ef94511be12a77464f18232 2024-11-21T19:49:03,315 DEBUG [RS_CLOSE_META-regionserver/7b96b36745c6:0-0 {event_type=M_RS_CLOSE_META}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0b774921-3780-c936-4fe1-1d1dcb4ce861/545083c6-d774-d2ca-7409-0fbc2c22cf1a/data/hbase/meta/1588230740/info/520f9ebe9ef94511be12a77464f18232, delay=947 ms 2024-11-21T19:49:03,316 INFO [regionserver/7b96b36745c6:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-21T19:49:03,316 INFO [RS_CLOSE_META-regionserver/7b96b36745c6:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0b774921-3780-c936-4fe1-1d1dcb4ce861/545083c6-d774-d2ca-7409-0fbc2c22cf1a/data/hbase/meta/1588230740/info/520f9ebe9ef94511be12a77464f18232, entries=8, sequenceid=19, filesize=6.6 K 2024-11-21T19:49:03,317 INFO [RS_CLOSE_META-regionserver/7b96b36745c6:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~1.21 KB/1238, heapSize ~2.03 KB/2080, currentSize=0 B/0 for 1588230740 in 32ms, sequenceid=19, compaction requested=false 2024-11-21T19:49:03,318 DEBUG [StoreFileCloser-1588230740-info-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0b774921-3780-c936-4fe1-1d1dcb4ce861/545083c6-d774-d2ca-7409-0fbc2c22cf1a/data/hbase/meta/1588230740/info/52857c24a1964413a4d2a29dac0380c6 2024-11-21T19:49:03,318 DEBUG [StoreFileCloser-1588230740-info-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0b774921-3780-c936-4fe1-1d1dcb4ce861/545083c6-d774-d2ca-7409-0fbc2c22cf1a/data/hbase/meta/1588230740/info/520f9ebe9ef94511be12a77464f18232 2024-11-21T19:49:03,318 DEBUG [StoreFileCloser-1588230740-ns-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0b774921-3780-c936-4fe1-1d1dcb4ce861/545083c6-d774-d2ca-7409-0fbc2c22cf1a/data/hbase/meta/1588230740/ns/b1cb9e8185164438bba7e9d2c46d5af4 2024-11-21T19:49:03,319 DEBUG [StoreFileCloser-1588230740-table-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0b774921-3780-c936-4fe1-1d1dcb4ce861/545083c6-d774-d2ca-7409-0fbc2c22cf1a/data/hbase/meta/1588230740/table/bac66e20f4f54369b69f7de0858f7356 2024-11-21T19:49:03,323 DEBUG [RS_CLOSE_META-regionserver/7b96b36745c6:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0b774921-3780-c936-4fe1-1d1dcb4ce861/545083c6-d774-d2ca-7409-0fbc2c22cf1a/data/hbase/meta/1588230740/recovered.edits/22.seqid, newMaxSeqId=22, maxSeqId=14 2024-11-21T19:49:03,323 DEBUG [RS_CLOSE_META-regionserver/7b96b36745c6:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-21T19:49:03,323 INFO [RS_CLOSE_META-regionserver/7b96b36745c6:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-21T19:49:03,323 DEBUG [RS_CLOSE_META-regionserver/7b96b36745c6:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1732218543285Running coprocessor pre-close hooks at 1732218543285Disabling compacts and flushes for region at 1732218543285Disabling writes for close at 1732218543285Obtaining lock to block concurrent updates at 1732218543285Preparing flush snapshotting stores in 1588230740 at 1732218543285Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=1238, getHeapSize=2800, getOffHeapSize=0, getCellsCount=8 at 1732218543286 (+1 ms)Flushing stores of hbase:meta,,1.1588230740 at 1732218543286Flushing 1588230740/info: creating writer at 1732218543286Flushing 1588230740/info: appending metadata at 1732218543311 (+25 ms)Flushing 1588230740/info: closing flushed file at 1732218543311Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@2194dc92: reopening flushed file at 1732218543314 (+3 ms)Finished flush of dataSize ~1.21 KB/1238, heapSize ~2.03 KB/2080, currentSize=0 B/0 for 1588230740 in 32ms, sequenceid=19, compaction requested=false at 1732218543317 (+3 ms)Writing region close event to WAL at 1732218543319 (+2 ms)Running coprocessor post-close hooks at 1732218543323 (+4 ms)Closed at 1732218543323 2024-11-21T19:49:03,324 DEBUG [RS_CLOSE_META-regionserver/7b96b36745c6:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-11-21T19:49:03,326 INFO [regionserver/7b96b36745c6:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-21T19:49:03,392 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44887-0x10116d8419f0003, quorum=127.0.0.1:54341, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-21T19:49:03,392 INFO [RS:2;7b96b36745c6:44887 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-21T19:49:03,392 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44887-0x10116d8419f0003, quorum=127.0.0.1:54341, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-21T19:49:03,393 INFO [RS:2;7b96b36745c6:44887 {}] regionserver.HRegionServer(1031): Exiting; stopping=7b96b36745c6,44887,1732218542659; zookeeper connection closed. 2024-11-21T19:49:03,393 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@3c968869 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@3c968869 2024-11-21T19:49:03,485 INFO [RS:0;7b96b36745c6:43071 {}] regionserver.HRegionServer(976): stopping server 7b96b36745c6,43071,1732218539369; all regions closed. 2024-11-21T19:49:03,487 DEBUG [RS:0;7b96b36745c6:43071 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0b774921-3780-c936-4fe1-1d1dcb4ce861/545083c6-d774-d2ca-7409-0fbc2c22cf1a/oldWALs 2024-11-21T19:49:03,487 INFO [RS:0;7b96b36745c6:43071 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 7b96b36745c6%2C43071%2C1732218539369.meta:.meta(num 1732218542701) 2024-11-21T19:49:03,488 DEBUG [RS:0;7b96b36745c6:43071 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0b774921-3780-c936-4fe1-1d1dcb4ce861/545083c6-d774-d2ca-7409-0fbc2c22cf1a/oldWALs 2024-11-21T19:49:03,488 INFO [RS:0;7b96b36745c6:43071 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 7b96b36745c6%2C43071%2C1732218539369:(num 1732218539958) 2024-11-21T19:49:03,488 DEBUG [RS:0;7b96b36745c6:43071 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T19:49:03,488 INFO [RS:0;7b96b36745c6:43071 {}] regionserver.LeaseManager(133): Closed leases 2024-11-21T19:49:03,488 INFO [RS:0;7b96b36745c6:43071 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-21T19:49:03,489 INFO [RS:0;7b96b36745c6:43071 {}] hbase.ChoreService(370): Chore service for: regionserver/7b96b36745c6:0 had [ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS] on shutdown 2024-11-21T19:49:03,489 INFO [RS:0;7b96b36745c6:43071 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-21T19:49:03,489 INFO [regionserver/7b96b36745c6:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-21T19:49:03,489 INFO [RS:0;7b96b36745c6:43071 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:43071 2024-11-21T19:49:03,492 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43071-0x10116d8419f0001, quorum=127.0.0.1:54341, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/7b96b36745c6,43071,1732218539369 2024-11-21T19:49:03,492 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44549-0x10116d8419f0000, quorum=127.0.0.1:54341, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-21T19:49:03,492 INFO [RS:0;7b96b36745c6:43071 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-21T19:49:03,494 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [7b96b36745c6,43071,1732218539369] 2024-11-21T19:49:03,495 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/7b96b36745c6,43071,1732218539369 already deleted, retry=false 2024-11-21T19:49:03,495 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; 7b96b36745c6,43071,1732218539369 expired; onlineServers=0 2024-11-21T19:49:03,496 INFO [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master '7b96b36745c6,44549,1732218539289' ***** 2024-11-21T19:49:03,496 INFO [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0 2024-11-21T19:49:03,496 INFO [M:0;7b96b36745c6:44549 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-21T19:49:03,496 INFO [M:0;7b96b36745c6:44549 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-21T19:49:03,496 DEBUG [M:0;7b96b36745c6:44549 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2024-11-21T19:49:03,496 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2024-11-21T19:49:03,496 DEBUG [M:0;7b96b36745c6:44549 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2024-11-21T19:49:03,496 DEBUG [master/7b96b36745c6:0:becomeActiveMaster-HFileCleaner.small.0-1732218539693 {}] cleaner.HFileCleaner(306): Exit Thread[master/7b96b36745c6:0:becomeActiveMaster-HFileCleaner.small.0-1732218539693,5,FailOnTimeoutGroup] 2024-11-21T19:49:03,496 DEBUG [master/7b96b36745c6:0:becomeActiveMaster-HFileCleaner.large.0-1732218539689 {}] cleaner.HFileCleaner(306): Exit Thread[master/7b96b36745c6:0:becomeActiveMaster-HFileCleaner.large.0-1732218539689,5,FailOnTimeoutGroup] 2024-11-21T19:49:03,496 INFO [M:0;7b96b36745c6:44549 {}] hbase.ChoreService(370): Chore service for: master/7b96b36745c6:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown 2024-11-21T19:49:03,497 INFO [M:0;7b96b36745c6:44549 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-21T19:49:03,497 DEBUG [M:0;7b96b36745c6:44549 {}] master.HMaster(1795): Stopping service threads 2024-11-21T19:49:03,497 INFO [M:0;7b96b36745c6:44549 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2024-11-21T19:49:03,497 INFO [M:0;7b96b36745c6:44549 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-21T19:49:03,497 INFO [M:0;7b96b36745c6:44549 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-21T19:49:03,497 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2024-11-21T19:49:03,498 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44549-0x10116d8419f0000, quorum=127.0.0.1:54341, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-11-21T19:49:03,498 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44549-0x10116d8419f0000, quorum=127.0.0.1:54341, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-21T19:49:03,498 DEBUG [M:0;7b96b36745c6:44549 {}] zookeeper.ZKUtil(347): master:44549-0x10116d8419f0000, quorum=127.0.0.1:54341, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error) 2024-11-21T19:49:03,498 WARN [M:0;7b96b36745c6:44549 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null 2024-11-21T19:49:03,498 INFO [M:0;7b96b36745c6:44549 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0b774921-3780-c936-4fe1-1d1dcb4ce861/545083c6-d774-d2ca-7409-0fbc2c22cf1a/.lastflushedseqids 2024-11-21T19:49:03,505 INFO [M:0;7b96b36745c6:44549 {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-21T19:49:03,505 INFO [M:0;7b96b36745c6:44549 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2024-11-21T19:49:03,505 DEBUG [M:0;7b96b36745c6:44549 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-21T19:49:03,505 INFO [M:0;7b96b36745c6:44549 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-21T19:49:03,505 DEBUG [M:0;7b96b36745c6:44549 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-21T19:49:03,506 DEBUG [M:0;7b96b36745c6:44549 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-21T19:49:03,506 DEBUG [M:0;7b96b36745c6:44549 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-21T19:49:03,506 INFO [M:0;7b96b36745c6:44549 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=39.50 KB heapSize=49.91 KB 2024-11-21T19:49:03,516 DEBUG [M:0;7b96b36745c6:44549 {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on be1615190db34599877ca31efc19883b 2024-11-21T19:49:03,530 DEBUG [M:0;7b96b36745c6:44549 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0b774921-3780-c936-4fe1-1d1dcb4ce861/545083c6-d774-d2ca-7409-0fbc2c22cf1a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/be1615190db34599877ca31efc19883b is 82, key is hbase:meta,,1/info:regioninfo/1732218542745/Put/seqid=0 2024-11-21T19:49:03,531 INFO [M:0;7b96b36745c6:44549 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1008 B at sequenceid=102 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0b774921-3780-c936-4fe1-1d1dcb4ce861/545083c6-d774-d2ca-7409-0fbc2c22cf1a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/be1615190db34599877ca31efc19883b 2024-11-21T19:49:03,539 DEBUG [M:0;7b96b36745c6:44549 {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 0ac74dcafddb4892bfd30a5e813ee1f2 2024-11-21T19:49:03,553 DEBUG [M:0;7b96b36745c6:44549 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0b774921-3780-c936-4fe1-1d1dcb4ce861/545083c6-d774-d2ca-7409-0fbc2c22cf1a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/0ac74dcafddb4892bfd30a5e813ee1f2 is 782, key is \x00\x00\x00\x00\x00\x00\x00\x04/proc:d/1732218540845/Put/seqid=0 2024-11-21T19:49:03,553 INFO [M:0;7b96b36745c6:44549 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.27 KB at sequenceid=102 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0b774921-3780-c936-4fe1-1d1dcb4ce861/545083c6-d774-d2ca-7409-0fbc2c22cf1a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/0ac74dcafddb4892bfd30a5e813ee1f2 2024-11-21T19:49:03,561 DEBUG [M:0;7b96b36745c6:44549 {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 9d4766aebc5642dd8aac52b0bd4b3506 2024-11-21T19:49:03,574 DEBUG [M:0;7b96b36745c6:44549 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0b774921-3780-c936-4fe1-1d1dcb4ce861/545083c6-d774-d2ca-7409-0fbc2c22cf1a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/9d4766aebc5642dd8aac52b0bd4b3506 is 69, key is 7b96b36745c6,43071,1732218539369/rs:state/1732218539784/Put/seqid=0 2024-11-21T19:49:03,575 INFO [M:0;7b96b36745c6:44549 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=249 B at sequenceid=102 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0b774921-3780-c936-4fe1-1d1dcb4ce861/545083c6-d774-d2ca-7409-0fbc2c22cf1a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/9d4766aebc5642dd8aac52b0bd4b3506 2024-11-21T19:49:03,577 INFO [M:0;7b96b36745c6:44549 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 9d4766aebc5642dd8aac52b0bd4b3506 2024-11-21T19:49:03,577 DEBUG [M:0;7b96b36745c6:44549 {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0b774921-3780-c936-4fe1-1d1dcb4ce861/545083c6-d774-d2ca-7409-0fbc2c22cf1a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/be1615190db34599877ca31efc19883b as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0b774921-3780-c936-4fe1-1d1dcb4ce861/545083c6-d774-d2ca-7409-0fbc2c22cf1a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/be1615190db34599877ca31efc19883b 2024-11-21T19:49:03,579 INFO [M:0;7b96b36745c6:44549 {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0b774921-3780-c936-4fe1-1d1dcb4ce861/545083c6-d774-d2ca-7409-0fbc2c22cf1a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/be1615190db34599877ca31efc19883b, entries=14, sequenceid=102, filesize=5.9 K 2024-11-21T19:49:03,580 DEBUG [M:0;7b96b36745c6:44549 {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0b774921-3780-c936-4fe1-1d1dcb4ce861/545083c6-d774-d2ca-7409-0fbc2c22cf1a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/0ac74dcafddb4892bfd30a5e813ee1f2 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0b774921-3780-c936-4fe1-1d1dcb4ce861/545083c6-d774-d2ca-7409-0fbc2c22cf1a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/0ac74dcafddb4892bfd30a5e813ee1f2 2024-11-21T19:49:03,581 INFO [M:0;7b96b36745c6:44549 {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0b774921-3780-c936-4fe1-1d1dcb4ce861/545083c6-d774-d2ca-7409-0fbc2c22cf1a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/0ac74dcafddb4892bfd30a5e813ee1f2, entries=11, sequenceid=102, filesize=6.5 K 2024-11-21T19:49:03,581 DEBUG [M:0;7b96b36745c6:44549 {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0b774921-3780-c936-4fe1-1d1dcb4ce861/545083c6-d774-d2ca-7409-0fbc2c22cf1a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/9d4766aebc5642dd8aac52b0bd4b3506 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0b774921-3780-c936-4fe1-1d1dcb4ce861/545083c6-d774-d2ca-7409-0fbc2c22cf1a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/9d4766aebc5642dd8aac52b0bd4b3506 2024-11-21T19:49:03,583 INFO [M:0;7b96b36745c6:44549 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 9d4766aebc5642dd8aac52b0bd4b3506 2024-11-21T19:49:03,583 INFO [M:0;7b96b36745c6:44549 {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0b774921-3780-c936-4fe1-1d1dcb4ce861/545083c6-d774-d2ca-7409-0fbc2c22cf1a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/9d4766aebc5642dd8aac52b0bd4b3506, entries=3, sequenceid=102, filesize=5.3 K 2024-11-21T19:49:03,584 INFO [M:0;7b96b36745c6:44549 {}] regionserver.HRegion(3140): Finished flush of dataSize ~39.50 KB/40448, heapSize ~49.61 KB/50800, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 78ms, sequenceid=102, compaction requested=false 2024-11-21T19:49:03,586 INFO [M:0;7b96b36745c6:44549 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-21T19:49:03,586 DEBUG [M:0;7b96b36745c6:44549 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1732218543505Disabling compacts and flushes for region at 1732218543505Disabling writes for close at 1732218543506 (+1 ms)Obtaining lock to block concurrent updates at 1732218543506Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1732218543506Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=40448, getHeapSize=51040, getOffHeapSize=0, getCellsCount=127 at 1732218543506Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1732218543507 (+1 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1732218543507Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1732218543530 (+23 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1732218543530Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1732218543533 (+3 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1732218543552 (+19 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1732218543552Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1732218543555 (+3 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1732218543573 (+18 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1732218543573Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@74884875: reopening flushed file at 1732218543577 (+4 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@1f048f99: reopening flushed file at 1732218543579 (+2 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@7d337cbd: reopening flushed file at 1732218543581 (+2 ms)Finished flush of dataSize ~39.50 KB/40448, heapSize ~49.61 KB/50800, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 78ms, sequenceid=102, compaction requested=false at 1732218543584 (+3 ms)Writing region close event to WAL at 1732218543586 (+2 ms)Closed at 1732218543586 2024-11-21T19:49:03,588 INFO [M:0;7b96b36745c6:44549 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down. 2024-11-21T19:49:03,588 INFO [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-21T19:49:03,588 INFO [M:0;7b96b36745c6:44549 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:44549 2024-11-21T19:49:03,589 INFO [M:0;7b96b36745c6:44549 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-21T19:49:03,594 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43071-0x10116d8419f0001, quorum=127.0.0.1:54341, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-21T19:49:03,594 INFO [RS:0;7b96b36745c6:43071 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-21T19:49:03,594 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43071-0x10116d8419f0001, quorum=127.0.0.1:54341, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-21T19:49:03,594 INFO [RS:0;7b96b36745c6:43071 {}] regionserver.HRegionServer(1031): Exiting; stopping=7b96b36745c6,43071,1732218539369; zookeeper connection closed. 2024-11-21T19:49:03,595 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@11dbad1b {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@11dbad1b 2024-11-21T19:49:03,595 INFO [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 3 regionserver(s) complete 2024-11-21T19:49:03,691 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44549-0x10116d8419f0000, quorum=127.0.0.1:54341, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-21T19:49:03,691 INFO [M:0;7b96b36745c6:44549 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-21T19:49:03,691 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44549-0x10116d8419f0000, quorum=127.0.0.1:54341, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-21T19:49:03,693 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers 2024-11-21T19:49:03,694 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down 2024-11-21T19:49:03,694 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-21T19:49:03,701 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: io.hfile.TestBlockEvictionOnRegionMovement#testBlockEvictionOnGracefulStop Thread=69 (was 49) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-7-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-8-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HMaster-EventLoopGroup-6-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-7-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HMaster-EventLoopGroup-6-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HMaster-EventLoopGroup-6-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-8 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-9-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-9-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-6 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-8-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-7-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-8-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-5 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-7 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=382 (was 346) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=441 (was 480), ProcessCount=11 (was 11), AvailableMemoryMB=12082 (was 12268)