2024-11-15 21:58:55,397 main DEBUG Apache Log4j Core 2.17.2 initializing configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@5b38c1ec 2024-11-15 21:58:55,413 main DEBUG Took 0.013340 seconds to load 1 plugins from package org.apache.hadoop.hbase.logging 2024-11-15 21:58:55,413 main DEBUG PluginManager 'Core' found 129 plugins 2024-11-15 21:58:55,414 main DEBUG PluginManager 'Level' found 0 plugins 2024-11-15 21:58:55,415 main DEBUG PluginManager 'Lookup' found 16 plugins 2024-11-15 21:58:55,417 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-15 21:58:55,429 main DEBUG PluginManager 'TypeConverter' found 26 plugins 2024-11-15 21:58:55,476 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.util.MBeans", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-15 21:58:55,482 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-15 21:58:55,483 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.logging.TestJul2Slf4j", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-15 21:58:55,483 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-15 21:58:55,484 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.zookeeper", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-15 21:58:55,484 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-15 21:58:55,486 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSinkAdapter", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-15 21:58:55,486 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-15 21:58:55,487 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSystemImpl", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-15 21:58:55,487 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-15 21:58:55,488 main DEBUG LoggerConfig$Builder(additivity="false", level="WARN", levelAndRefs="null", name="org.apache.directory", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-15 21:58:55,489 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-15 21:58:55,490 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.ipc.FailedServers", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-15 21:58:55,490 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-15 21:58:55,491 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsConfig", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-15 21:58:55,492 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-15 21:58:55,492 main DEBUG LoggerConfig$Builder(additivity="null", level="INFO", levelAndRefs="null", name="org.apache.hadoop.hbase.ScheduledChore", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-15 21:58:55,493 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-15 21:58:55,493 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.regionserver.RSRpcServices", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-15 21:58:55,494 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-15 21:58:55,494 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-15 21:58:55,498 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-15 21:58:55,499 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-15 21:58:55,500 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-15 21:58:55,500 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hbase.thirdparty.io.netty.channel", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-15 21:58:55,501 main DEBUG Building Plugin[name=root, class=org.apache.logging.log4j.core.config.LoggerConfig$RootLogger]. 2024-11-15 21:58:55,504 main DEBUG LoggerConfig$RootLogger$Builder(additivity="null", level="null", levelAndRefs="INFO,Console", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-15 21:58:55,506 main DEBUG Building Plugin[name=loggers, class=org.apache.logging.log4j.core.config.LoggersPlugin]. 2024-11-15 21:58:55,508 main DEBUG createLoggers(={org.apache.hadoop.metrics2.util.MBeans, org.apache.hadoop.hbase.logging.TestJul2Slf4j, org.apache.zookeeper, org.apache.hadoop.metrics2.impl.MetricsSinkAdapter, org.apache.hadoop.metrics2.impl.MetricsSystemImpl, org.apache.directory, org.apache.hadoop.hbase.ipc.FailedServers, org.apache.hadoop.metrics2.impl.MetricsConfig, org.apache.hadoop.hbase.ScheduledChore, org.apache.hadoop.hbase.regionserver.RSRpcServices, org.apache.hadoop, org.apache.hadoop.hbase, org.apache.hbase.thirdparty.io.netty.channel, root}) 2024-11-15 21:58:55,509 main DEBUG Building Plugin[name=layout, class=org.apache.logging.log4j.core.layout.PatternLayout]. 2024-11-15 21:58:55,510 main DEBUG PatternLayout$Builder(pattern="%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n", PatternSelector=null, Configuration(PropertiesConfig), Replace=null, charset="null", alwaysWriteExceptions="null", disableAnsi="null", noConsoleNoAnsi="null", header="null", footer="null") 2024-11-15 21:58:55,511 main DEBUG PluginManager 'Converter' found 47 plugins 2024-11-15 21:58:55,521 main DEBUG Building Plugin[name=appender, class=org.apache.hadoop.hbase.logging.HBaseTestAppender]. 2024-11-15 21:58:55,524 main DEBUG HBaseTestAppender$Builder(target="SYSTEM_ERR", maxSize="1G", bufferedIo="null", bufferSize="null", immediateFlush="null", ignoreExceptions="null", PatternLayout(%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n), name="Console", Configuration(PropertiesConfig), Filter=null, ={}) 2024-11-15 21:58:55,526 main DEBUG Starting HBaseTestOutputStreamManager SYSTEM_ERR 2024-11-15 21:58:55,526 main DEBUG Building Plugin[name=appenders, class=org.apache.logging.log4j.core.config.AppendersPlugin]. 2024-11-15 21:58:55,526 main DEBUG createAppenders(={Console}) 2024-11-15 21:58:55,527 main DEBUG Configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@5b38c1ec initialized 2024-11-15 21:58:55,527 main DEBUG Starting configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@5b38c1ec 2024-11-15 21:58:55,528 main DEBUG Started configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@5b38c1ec OK. 2024-11-15 21:58:55,528 main DEBUG Shutting down OutputStreamManager SYSTEM_OUT.false.false-1 2024-11-15 21:58:55,528 main DEBUG OutputStream closed 2024-11-15 21:58:55,529 main DEBUG Shut down OutputStreamManager SYSTEM_OUT.false.false-1, all resources released: true 2024-11-15 21:58:55,529 main DEBUG Appender DefaultConsole-1 stopped with status true 2024-11-15 21:58:55,529 main DEBUG Stopped org.apache.logging.log4j.core.config.DefaultConfiguration@bae7dc0 OK 2024-11-15 21:58:55,626 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6 2024-11-15 21:58:55,629 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=StatusLogger 2024-11-15 21:58:55,630 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=ContextSelector 2024-11-15 21:58:55,632 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name= 2024-11-15 21:58:55,632 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.directory 2024-11-15 21:58:55,633 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSinkAdapter 2024-11-15 21:58:55,633 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.zookeeper 2024-11-15 21:58:55,634 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.logging.TestJul2Slf4j 2024-11-15 21:58:55,634 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSystemImpl 2024-11-15 21:58:55,635 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.util.MBeans 2024-11-15 21:58:55,635 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase 2024-11-15 21:58:55,636 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop 2024-11-15 21:58:55,636 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ipc.FailedServers 2024-11-15 21:58:55,637 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.regionserver.RSRpcServices 2024-11-15 21:58:55,637 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsConfig 2024-11-15 21:58:55,638 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hbase.thirdparty.io.netty.channel 2024-11-15 21:58:55,638 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ScheduledChore 2024-11-15 21:58:55,639 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Appenders,name=Console 2024-11-15 21:58:55,643 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-15 21:58:55,643 main DEBUG Reconfiguration complete for context[name=1dbd16a6] at URI jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-logging/target/hbase-logging-4.0.0-alpha-1-SNAPSHOT-tests.jar!/log4j2.properties (org.apache.logging.log4j.core.LoggerContext@2b76ff4e) with optional ClassLoader: null 2024-11-15 21:58:55,643 main DEBUG Shutdown hook enabled. Registering a new one. 2024-11-15 21:58:55,644 main DEBUG LoggerContext[name=1dbd16a6, org.apache.logging.log4j.core.LoggerContext@2b76ff4e] started OK. 2024-11-15T21:58:55,661 INFO [main {}] hbase.HBaseClassTestRule(94): Test class org.apache.hadoop.hbase.io.hfile.TestBlockEvictionOnRegionMovement timeout: 13 mins 2024-11-15 21:58:55,663 main DEBUG AsyncLogger.ThreadNameStrategy=UNCACHED (user specified null, default is UNCACHED) 2024-11-15 21:58:55,664 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-15T21:58:55,930 DEBUG [main {}] hbase.HBaseTestingUtil(323): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3be1b200-2a9c-651b-3f6b-ac964062b7d6 2024-11-15T21:58:55,954 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: io.hfile.TestBlockEvictionOnRegionMovement#testBlockEvictionOnRegionMove Thread=12, OpenFileDescriptor=287, MaxFileDescriptor=1048576, SystemLoadAverage=1083, ProcessCount=11, AvailableMemoryMB=3331 2024-11-15T21:58:56,051 WARN [Time-limited test {}] util.NativeCodeLoader(60): Unable to load native-hadoop library for your platform... using builtin-java classes where applicable 2024-11-15T21:58:56,151 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-15T21:58:56,168 INFO [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3be1b200-2a9c-651b-3f6b-ac964062b7d6/cluster_1305362e-baa0-6ab0-5abc-1c34e3535d6b, deleteOnExit=true 2024-11-15T21:58:56,269 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3be1b200-2a9c-651b-3f6b-ac964062b7d6/cluster_1305362e-baa0-6ab0-5abc-1c34e3535d6b/zookeeper_0, clientPort=62797, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3be1b200-2a9c-651b-3f6b-ac964062b7d6/cluster_1305362e-baa0-6ab0-5abc-1c34e3535d6b/zookeeper_0/version-2, dataDirSize=457 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3be1b200-2a9c-651b-3f6b-ac964062b7d6/cluster_1305362e-baa0-6ab0-5abc-1c34e3535d6b/zookeeper_0/version-2, dataLogSize=457 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0 2024-11-15T21:58:56,282 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=62797 2024-11-15T21:58:56,282 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-15T21:58:56,378 INFO [Time-limited test {}] util.FSUtils(489): Created version file at /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3be1b200-2a9c-651b-3f6b-ac964062b7d6/f4b02001-5f5c-c384-44d5-c547c79a7f1b with version=8 2024-11-15T21:58:56,379 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1139): Setting hbase.fs.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3be1b200-2a9c-651b-3f6b-ac964062b7d6/f4b02001-5f5c-c384-44d5-c547c79a7f1b/hbase-staging 2024-11-15T21:58:56,480 DEBUG [Time-limited test {}] channel.MultithreadEventLoopGroup(44): -Dio.netty.eventLoopThreads: 16 2024-11-15T21:58:56,778 INFO [Time-limited test {}] client.ConnectionUtils(128): master/ae3311ea5ccb:0 server-side Connection retries=45 2024-11-15T21:58:56,794 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-15T21:58:56,797 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-15T21:58:56,807 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-15T21:58:56,808 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-15T21:58:56,808 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-15T21:58:56,999 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService 2024-11-15T21:58:57,092 WARN [Time-limited test {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-15T21:58:57,130 INFO [Time-limited test {}] metrics.MetricRegistriesLoader(60): Loaded MetricRegistries class org.apache.hadoop.hbase.metrics.impl.MetricRegistriesImpl 2024-11-15T21:58:57,141 DEBUG [Time-limited test {}] util.ClassSize(228): Using Unsafe to estimate memory layout 2024-11-15T21:58:57,145 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-15T21:58:57,190 DEBUG [Time-limited test {}] channel.DefaultChannelId(84): -Dio.netty.processId: 21893 (auto-detected) 2024-11-15T21:58:57,191 DEBUG [Time-limited test {}] channel.DefaultChannelId(106): -Dio.netty.machineId: 02:42:ac:ff:fe:11:00:03 (auto-detected) 2024-11-15T21:58:57,215 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.3:40769 2024-11-15T21:58:57,239 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:40769 connecting to ZooKeeper ensemble=127.0.0.1:62797 2024-11-15T21:58:57,276 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:407690x0, quorum=127.0.0.1:62797, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-15T21:58:57,279 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:40769-0x100f86912970000 connected 2024-11-15T21:58:57,307 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-15T21:58:57,308 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-15T21:58:57,316 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:40769-0x100f86912970000, quorum=127.0.0.1:62797, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-15T21:58:57,320 INFO [Time-limited test {}] master.HMaster(525): hbase.rootdir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3be1b200-2a9c-651b-3f6b-ac964062b7d6/f4b02001-5f5c-c384-44d5-c547c79a7f1b, hbase.cluster.distributed=false 2024-11-15T21:58:57,352 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:40769-0x100f86912970000, quorum=127.0.0.1:62797, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-15T21:58:57,360 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=40769 2024-11-15T21:58:57,360 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=40769 2024-11-15T21:58:57,361 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=40769 2024-11-15T21:58:57,369 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=40769 2024-11-15T21:58:57,370 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=40769 2024-11-15T21:58:57,460 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/ae3311ea5ccb:0 server-side Connection retries=45 2024-11-15T21:58:57,462 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-15T21:58:57,462 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-15T21:58:57,463 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-15T21:58:57,463 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-15T21:58:57,463 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-15T21:58:57,466 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-15T21:58:57,468 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-15T21:58:57,469 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.3:35509 2024-11-15T21:58:57,471 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:35509 connecting to ZooKeeper ensemble=127.0.0.1:62797 2024-11-15T21:58:57,472 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-15T21:58:57,473 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-15T21:58:57,477 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:355090x0, quorum=127.0.0.1:62797, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-15T21:58:57,478 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:35509-0x100f86912970001 connected 2024-11-15T21:58:57,479 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:35509-0x100f86912970001, quorum=127.0.0.1:62797, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-15T21:58:57,484 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-15T21:58:57,497 INFO [Time-limited test {}] bucket.FileIOEngine(99): Allocating cache 400 MB, on the path:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3be1b200-2a9c-651b-3f6b-ac964062b7d6/bucket.cache 2024-11-15T21:58:57,503 INFO [Time-limited test {}] bucket.BucketCache(364): Instantiating BucketCache with acceptableFactor: 0.95, minFactor: 0.85, extraFreeFactor: 0.1, singleFactor: 0.25, multiFactor: 0.5, memoryFactor: 0.25, useStrongRef: false 2024-11-15T21:58:57,504 INFO [Time-limited test {}] bucket.BucketCache(478): Starting BucketCachePersisterThread 2024-11-15T21:58:57,505 INFO [Time-limited test {}] bucket.BucketCachePersister(38): BucketCachePersister started with interval: 100 2024-11-15T21:58:57,507 INFO [Thread-10 {}] bucket.BucketCache(1530): Started retrieving bucket cache from file 2024-11-15T21:58:57,507 WARN [Thread-10 {}] bucket.BucketCache(1533): Persistence file missing! It's ok if it's first run after enabling persistent cache. 2024-11-15T21:58:57,514 INFO [Time-limited test {}] bucket.BucketCache(403): Started bucket cache; ioengine=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3be1b200-2a9c-651b-3f6b-ac964062b7d6/bucket.cache, capacity=400 MB, blockSize=64 KB, writerThreadNum=3, writerQLen=64, persistencePath=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3be1b200-2a9c-651b-3f6b-ac964062b7d6/bucket.persistence, bucketAllocator=org.apache.hadoop.hbase.io.hfile.bucket.BucketAllocator 2024-11-15T21:58:57,519 INFO [Thread-10 {}] bucket.BucketAllocator(343): Cache totalSize=418148352, buckets=199, bucket capacity=2101248=(4*525312)=(FEWEST_ITEMS_IN_BUCKET*(largest configured bucketcache size)) 2024-11-15T21:58:57,519 INFO [Thread-10 {}] bucket.BucketCache(413): Persistent bucket cache recovery from /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3be1b200-2a9c-651b-3f6b-ac964062b7d6/bucket.persistence is complete. 2024-11-15T21:58:57,521 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-15T21:58:57,524 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:35509-0x100f86912970001, quorum=127.0.0.1:62797, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-15T21:58:57,529 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:35509-0x100f86912970001, quorum=127.0.0.1:62797, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-15T21:58:57,531 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=35509 2024-11-15T21:58:57,531 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=35509 2024-11-15T21:58:57,532 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=35509 2024-11-15T21:58:57,538 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=35509 2024-11-15T21:58:57,538 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=35509 2024-11-15T21:58:57,563 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/ae3311ea5ccb:0 server-side Connection retries=45 2024-11-15T21:58:57,563 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-15T21:58:57,564 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-15T21:58:57,564 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-15T21:58:57,564 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-15T21:58:57,564 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-15T21:58:57,565 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-15T21:58:57,565 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-15T21:58:57,567 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.3:32843 2024-11-15T21:58:57,570 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:32843 connecting to ZooKeeper ensemble=127.0.0.1:62797 2024-11-15T21:58:57,571 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-15T21:58:57,571 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-15T21:58:57,583 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:328430x0, quorum=127.0.0.1:62797, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-15T21:58:57,583 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:328430x0, quorum=127.0.0.1:62797, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-15T21:58:57,584 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-15T21:58:57,584 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:32843-0x100f86912970002 connected 2024-11-15T21:58:57,589 INFO [Time-limited test {}] bucket.FileIOEngine(99): Allocating cache 400 MB, on the path:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3be1b200-2a9c-651b-3f6b-ac964062b7d6/bucket.cache 2024-11-15T21:58:57,590 INFO [Time-limited test {}] bucket.BucketCache(364): Instantiating BucketCache with acceptableFactor: 0.95, minFactor: 0.85, extraFreeFactor: 0.1, singleFactor: 0.25, multiFactor: 0.5, memoryFactor: 0.25, useStrongRef: false 2024-11-15T21:58:57,590 INFO [Time-limited test {}] bucket.BucketCache(478): Starting BucketCachePersisterThread 2024-11-15T21:58:57,590 INFO [Time-limited test {}] bucket.BucketCachePersister(38): BucketCachePersister started with interval: 100 2024-11-15T21:58:57,595 INFO [Thread-11 {}] bucket.BucketCache(1530): Started retrieving bucket cache from file 2024-11-15T21:58:57,595 INFO [Time-limited test {}] bucket.BucketCache(403): Started bucket cache; ioengine=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3be1b200-2a9c-651b-3f6b-ac964062b7d6/bucket.cache, capacity=400 MB, blockSize=64 KB, writerThreadNum=3, writerQLen=64, persistencePath=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3be1b200-2a9c-651b-3f6b-ac964062b7d6/bucket.persistence, bucketAllocator=org.apache.hadoop.hbase.io.hfile.bucket.BucketAllocator 2024-11-15T21:58:57,595 WARN [Thread-11 {}] bucket.BucketCache(1533): Persistence file missing! It's ok if it's first run after enabling persistent cache. 2024-11-15T21:58:57,596 INFO [Thread-11 {}] bucket.BucketAllocator(343): Cache totalSize=418148352, buckets=199, bucket capacity=2101248=(4*525312)=(FEWEST_ITEMS_IN_BUCKET*(largest configured bucketcache size)) 2024-11-15T21:58:57,596 INFO [Thread-11 {}] bucket.BucketCache(413): Persistent bucket cache recovery from /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3be1b200-2a9c-651b-3f6b-ac964062b7d6/bucket.persistence is complete. 2024-11-15T21:58:57,597 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-15T21:58:57,602 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:32843-0x100f86912970002, quorum=127.0.0.1:62797, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-15T21:58:57,608 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:32843-0x100f86912970002, quorum=127.0.0.1:62797, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-15T21:58:57,609 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=32843 2024-11-15T21:58:57,610 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=32843 2024-11-15T21:58:57,616 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=32843 2024-11-15T21:58:57,641 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=32843 2024-11-15T21:58:57,643 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=32843 2024-11-15T21:58:57,668 DEBUG [M:0;ae3311ea5ccb:40769 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;ae3311ea5ccb:40769 2024-11-15T21:58:57,670 INFO [master/ae3311ea5ccb:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/ae3311ea5ccb,40769,1731707936561 2024-11-15T21:58:57,680 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40769-0x100f86912970000, quorum=127.0.0.1:62797, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-15T21:58:57,681 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:32843-0x100f86912970002, quorum=127.0.0.1:62797, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-15T21:58:57,682 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35509-0x100f86912970001, quorum=127.0.0.1:62797, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-15T21:58:57,692 DEBUG [master/ae3311ea5ccb:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:40769-0x100f86912970000, quorum=127.0.0.1:62797, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/ae3311ea5ccb,40769,1731707936561 2024-11-15T21:58:57,718 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:32843-0x100f86912970002, quorum=127.0.0.1:62797, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-15T21:58:57,718 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40769-0x100f86912970000, quorum=127.0.0.1:62797, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-15T21:58:57,718 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35509-0x100f86912970001, quorum=127.0.0.1:62797, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-15T21:58:57,718 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:32843-0x100f86912970002, quorum=127.0.0.1:62797, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-15T21:58:57,718 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35509-0x100f86912970001, quorum=127.0.0.1:62797, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-15T21:58:57,720 DEBUG [master/ae3311ea5ccb:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:40769-0x100f86912970000, quorum=127.0.0.1:62797, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-11-15T21:58:57,722 INFO [master/ae3311ea5ccb:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/ae3311ea5ccb,40769,1731707936561 from backup master directory 2024-11-15T21:58:57,725 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40769-0x100f86912970000, quorum=127.0.0.1:62797, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/ae3311ea5ccb,40769,1731707936561 2024-11-15T21:58:57,725 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:32843-0x100f86912970002, quorum=127.0.0.1:62797, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-15T21:58:57,725 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35509-0x100f86912970001, quorum=127.0.0.1:62797, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-15T21:58:57,726 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40769-0x100f86912970000, quorum=127.0.0.1:62797, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-15T21:58:57,727 WARN [master/ae3311ea5ccb:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-15T21:58:57,728 INFO [master/ae3311ea5ccb:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=ae3311ea5ccb,40769,1731707936561 2024-11-15T21:58:57,731 INFO [master/ae3311ea5ccb:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating data MemStoreChunkPool with chunk size 2 MB, max count 396, initial count 0 2024-11-15T21:58:57,733 INFO [master/ae3311ea5ccb:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating index MemStoreChunkPool with chunk size 204.80 KB, max count 440, initial count 0 2024-11-15T21:58:57,747 DEBUG [master/ae3311ea5ccb:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3be1b200-2a9c-651b-3f6b-ac964062b7d6/f4b02001-5f5c-c384-44d5-c547c79a7f1b/hbase.id] with ID: 02d33e22-0b9b-43a7-a826-fb44bc49b2d6 2024-11-15T21:58:57,748 DEBUG [master/ae3311ea5ccb:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3be1b200-2a9c-651b-3f6b-ac964062b7d6/f4b02001-5f5c-c384-44d5-c547c79a7f1b/.tmp/hbase.id 2024-11-15T21:58:57,761 DEBUG [master/ae3311ea5ccb:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3be1b200-2a9c-651b-3f6b-ac964062b7d6/f4b02001-5f5c-c384-44d5-c547c79a7f1b/.tmp/hbase.id]:[file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3be1b200-2a9c-651b-3f6b-ac964062b7d6/f4b02001-5f5c-c384-44d5-c547c79a7f1b/hbase.id] 2024-11-15T21:58:57,827 DEBUG [master/ae3311ea5ccb:0:becomeActiveMaster {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-15T21:58:57,836 INFO [master/ae3311ea5ccb:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem. 2024-11-15T21:58:57,877 INFO [master/ae3311ea5ccb:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 38ms. 2024-11-15T21:58:57,885 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35509-0x100f86912970001, quorum=127.0.0.1:62797, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-15T21:58:57,888 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40769-0x100f86912970000, quorum=127.0.0.1:62797, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-15T21:58:57,894 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:32843-0x100f86912970002, quorum=127.0.0.1:62797, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-15T21:58:57,956 INFO [master/ae3311ea5ccb:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-15T21:58:57,961 INFO [master/ae3311ea5ccb:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-11-15T21:58:58,037 DEBUG [master/ae3311ea5ccb:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(244): No decryptEncryptedDataEncryptionKey method in DFSClient, should be hadoop version with HDFS-12396 java.lang.NoSuchMethodException: org.apache.hadoop.hdfs.DFSClient.decryptEncryptedDataEncryptionKey(org.apache.hadoop.fs.FileEncryptionInfo) at java.lang.Class.getDeclaredMethod(Class.java:2675) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelperWithoutHDFS12396(FanOutOneBlockAsyncDFSOutputSaslHelper.java:183) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelper(FanOutOneBlockAsyncDFSOutputSaslHelper.java:242) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.(FanOutOneBlockAsyncDFSOutputSaslHelper.java:253) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.lang.Class.forName0(Native Method) ~[?:?] at java.lang.Class.forName(Class.java:375) ~[?:?] at org.apache.hadoop.hbase.wal.AsyncFSWALProvider.load(AsyncFSWALProvider.java:150) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.getProviderClass(WALFactory.java:174) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:262) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:231) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegion.create(MasterRegion.java:400) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegionFactory.create(MasterRegionFactory.java:135) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.finishActiveMasterInitialization(HMaster.java:1003) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.startActiveMasterManager(HMaster.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.lambda$run$0(HMaster.java:613) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.lambda$tracedRunnable$2(TraceUtil.java:155) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T21:58:58,046 INFO [master/ae3311ea5ccb:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-15T21:58:58,171 INFO [master/ae3311ea5ccb:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3be1b200-2a9c-651b-3f6b-ac964062b7d6/f4b02001-5f5c-c384-44d5-c547c79a7f1b/MasterData/data/master/store 2024-11-15T21:58:58,224 INFO [master/ae3311ea5ccb:0:becomeActiveMaster {}] throttle.StoreHotnessProtector(112): StoreHotnessProtector is disabled. Set hbase.region.store.parallel.put.limit > 0 to enable, which may help mitigate load under heavy write pressure. 2024-11-15T21:58:58,228 DEBUG [master/ae3311ea5ccb:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-15T21:58:58,230 DEBUG [master/ae3311ea5ccb:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-15T21:58:58,230 INFO [master/ae3311ea5ccb:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-15T21:58:58,230 DEBUG [master/ae3311ea5ccb:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-15T21:58:58,233 DEBUG [master/ae3311ea5ccb:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 1 ms 2024-11-15T21:58:58,233 DEBUG [master/ae3311ea5ccb:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-15T21:58:58,233 INFO [master/ae3311ea5ccb:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-15T21:58:58,234 DEBUG [master/ae3311ea5ccb:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1731707938229Disabling compacts and flushes for region at 1731707938229Disabling writes for close at 1731707938233 (+4 ms)Writing region close event to WAL at 1731707938233Closed at 1731707938233 2024-11-15T21:58:58,235 WARN [master/ae3311ea5ccb:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3be1b200-2a9c-651b-3f6b-ac964062b7d6/f4b02001-5f5c-c384-44d5-c547c79a7f1b/MasterData/data/master/store/.initializing 2024-11-15T21:58:58,235 DEBUG [master/ae3311ea5ccb:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3be1b200-2a9c-651b-3f6b-ac964062b7d6/f4b02001-5f5c-c384-44d5-c547c79a7f1b/MasterData/WALs/ae3311ea5ccb,40769,1731707936561 2024-11-15T21:58:58,264 INFO [master/ae3311ea5ccb:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-15T21:58:58,282 INFO [master/ae3311ea5ccb:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=ae3311ea5ccb%2C40769%2C1731707936561, suffix=, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3be1b200-2a9c-651b-3f6b-ac964062b7d6/f4b02001-5f5c-c384-44d5-c547c79a7f1b/MasterData/WALs/ae3311ea5ccb,40769,1731707936561, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3be1b200-2a9c-651b-3f6b-ac964062b7d6/f4b02001-5f5c-c384-44d5-c547c79a7f1b/MasterData/oldWALs, maxLogs=10 2024-11-15T21:58:58,376 INFO [master/ae3311ea5ccb:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3be1b200-2a9c-651b-3f6b-ac964062b7d6/f4b02001-5f5c-c384-44d5-c547c79a7f1b/MasterData/WALs/ae3311ea5ccb,40769,1731707936561/ae3311ea5ccb%2C40769%2C1731707936561.1731707938289 2024-11-15T21:58:58,377 DEBUG [master/ae3311ea5ccb:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-11-15T21:58:58,378 DEBUG [master/ae3311ea5ccb:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-11-15T21:58:58,379 DEBUG [master/ae3311ea5ccb:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-15T21:58:58,384 DEBUG [master/ae3311ea5ccb:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-11-15T21:58:58,385 DEBUG [master/ae3311ea5ccb:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-11-15T21:58:58,435 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-11-15T21:58:58,468 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-11-15T21:58:58,472 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T21:58:58,474 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-15T21:58:58,475 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-11-15T21:58:58,481 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-11-15T21:58:58,481 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T21:58:58,481 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-15T21:58:58,482 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-11-15T21:58:58,486 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-11-15T21:58:58,487 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T21:58:58,487 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-15T21:58:58,488 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-11-15T21:58:58,492 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-11-15T21:58:58,492 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T21:58:58,492 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-15T21:58:58,493 DEBUG [master/ae3311ea5ccb:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682 2024-11-15T21:58:58,496 DEBUG [master/ae3311ea5ccb:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3be1b200-2a9c-651b-3f6b-ac964062b7d6/f4b02001-5f5c-c384-44d5-c547c79a7f1b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-15T21:58:58,497 DEBUG [master/ae3311ea5ccb:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3be1b200-2a9c-651b-3f6b-ac964062b7d6/f4b02001-5f5c-c384-44d5-c547c79a7f1b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-15T21:58:58,500 DEBUG [master/ae3311ea5ccb:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682 2024-11-15T21:58:58,501 DEBUG [master/ae3311ea5ccb:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682 2024-11-15T21:58:58,504 DEBUG [master/ae3311ea5ccb:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-15T21:58:58,506 DEBUG [master/ae3311ea5ccb:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-11-15T21:58:58,514 DEBUG [master/ae3311ea5ccb:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3be1b200-2a9c-651b-3f6b-ac964062b7d6/f4b02001-5f5c-c384-44d5-c547c79a7f1b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-15T21:58:58,516 INFO [master/ae3311ea5ccb:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=70612866, jitterRate=0.052213698625564575}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-15T21:58:58,524 DEBUG [master/ae3311ea5ccb:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682: Writing region info on filesystem at 1731707938404Initializing all the Stores at 1731707938404Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731707938405 (+1 ms)Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731707938406 (+1 ms)Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731707938407 (+1 ms)Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731707938407Cleaning up temporary data from old regions at 1731707938501 (+94 ms)Region opened successfully at 1731707938523 (+22 ms) 2024-11-15T21:58:58,525 INFO [master/ae3311ea5ccb:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-11-15T21:58:58,568 DEBUG [master/ae3311ea5ccb:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@75d6bb36, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=ae3311ea5ccb/172.17.0.3:0 2024-11-15T21:58:58,611 INFO [master/ae3311ea5ccb:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating... 2024-11-15T21:58:58,623 INFO [master/ae3311ea5ccb:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5 2024-11-15T21:58:58,624 INFO [master/ae3311ea5ccb:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50 2024-11-15T21:58:58,627 INFO [master/ae3311ea5ccb:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-15T21:58:58,627 INFO [master/ae3311ea5ccb:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-15T21:58:58,633 INFO [master/ae3311ea5ccb:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 5 msec 2024-11-15T21:58:58,633 INFO [master/ae3311ea5ccb:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-11-15T21:58:58,660 INFO [master/ae3311ea5ccb:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2024-11-15T21:58:58,671 DEBUG [master/ae3311ea5ccb:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:40769-0x100f86912970000, quorum=127.0.0.1:62797, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error) 2024-11-15T21:58:58,674 DEBUG [master/ae3311ea5ccb:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false 2024-11-15T21:58:58,677 INFO [master/ae3311ea5ccb:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2024-11-15T21:58:58,679 DEBUG [master/ae3311ea5ccb:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:40769-0x100f86912970000, quorum=127.0.0.1:62797, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error) 2024-11-15T21:58:58,681 DEBUG [master/ae3311ea5ccb:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false 2024-11-15T21:58:58,684 INFO [master/ae3311ea5ccb:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2024-11-15T21:58:58,690 DEBUG [master/ae3311ea5ccb:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:40769-0x100f86912970000, quorum=127.0.0.1:62797, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error) 2024-11-15T21:58:58,695 DEBUG [master/ae3311ea5ccb:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false 2024-11-15T21:58:58,698 DEBUG [master/ae3311ea5ccb:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:40769-0x100f86912970000, quorum=127.0.0.1:62797, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error) 2024-11-15T21:58:58,700 DEBUG [master/ae3311ea5ccb:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false 2024-11-15T21:58:58,734 DEBUG [master/ae3311ea5ccb:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:40769-0x100f86912970000, quorum=127.0.0.1:62797, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error) 2024-11-15T21:58:58,740 DEBUG [master/ae3311ea5ccb:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false 2024-11-15T21:58:58,759 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:32843-0x100f86912970002, quorum=127.0.0.1:62797, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-15T21:58:58,759 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40769-0x100f86912970000, quorum=127.0.0.1:62797, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-15T21:58:58,759 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35509-0x100f86912970001, quorum=127.0.0.1:62797, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-15T21:58:58,759 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:32843-0x100f86912970002, quorum=127.0.0.1:62797, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-15T21:58:58,759 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40769-0x100f86912970000, quorum=127.0.0.1:62797, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-15T21:58:58,759 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35509-0x100f86912970001, quorum=127.0.0.1:62797, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-15T21:58:58,770 INFO [master/ae3311ea5ccb:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=ae3311ea5ccb,40769,1731707936561, sessionid=0x100f86912970000, setting cluster-up flag (Was=false) 2024-11-15T21:58:58,796 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35509-0x100f86912970001, quorum=127.0.0.1:62797, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-15T21:58:58,796 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:32843-0x100f86912970002, quorum=127.0.0.1:62797, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-15T21:58:58,796 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40769-0x100f86912970000, quorum=127.0.0.1:62797, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-15T21:58:58,807 DEBUG [master/ae3311ea5ccb:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort 2024-11-15T21:58:58,809 DEBUG [master/ae3311ea5ccb:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=ae3311ea5ccb,40769,1731707936561 2024-11-15T21:58:58,817 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40769-0x100f86912970000, quorum=127.0.0.1:62797, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-15T21:58:58,817 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35509-0x100f86912970001, quorum=127.0.0.1:62797, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-15T21:58:58,821 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:32843-0x100f86912970002, quorum=127.0.0.1:62797, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-15T21:58:58,828 DEBUG [master/ae3311ea5ccb:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort 2024-11-15T21:58:58,830 DEBUG [master/ae3311ea5ccb:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=ae3311ea5ccb,40769,1731707936561 2024-11-15T21:58:58,838 INFO [master/ae3311ea5ccb:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3be1b200-2a9c-651b-3f6b-ac964062b7d6/f4b02001-5f5c-c384-44d5-c547c79a7f1b/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again 2024-11-15T21:58:58,863 INFO [RS:0;ae3311ea5ccb:35509 {}] regionserver.HRegionServer(746): ClusterId : 02d33e22-0b9b-43a7-a826-fb44bc49b2d6 2024-11-15T21:58:58,863 INFO [RS:1;ae3311ea5ccb:32843 {}] regionserver.HRegionServer(746): ClusterId : 02d33e22-0b9b-43a7-a826-fb44bc49b2d6 2024-11-15T21:58:58,867 DEBUG [RS:0;ae3311ea5ccb:35509 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-15T21:58:58,867 DEBUG [RS:1;ae3311ea5ccb:32843 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-15T21:58:58,879 DEBUG [RS:0;ae3311ea5ccb:35509 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-15T21:58:58,879 DEBUG [RS:0;ae3311ea5ccb:35509 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-15T21:58:58,886 DEBUG [RS:1;ae3311ea5ccb:32843 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-15T21:58:58,886 DEBUG [RS:1;ae3311ea5ccb:32843 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-15T21:58:58,887 DEBUG [RS:0;ae3311ea5ccb:35509 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-15T21:58:58,887 DEBUG [RS:0;ae3311ea5ccb:35509 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@654b722d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=ae3311ea5ccb/172.17.0.3:0 2024-11-15T21:58:58,890 DEBUG [RS:1;ae3311ea5ccb:32843 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-15T21:58:58,891 DEBUG [RS:1;ae3311ea5ccb:32843 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7b02e090, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=ae3311ea5ccb/172.17.0.3:0 2024-11-15T21:58:58,907 DEBUG [RS:0;ae3311ea5ccb:35509 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;ae3311ea5ccb:35509 2024-11-15T21:58:58,911 DEBUG [RS:1;ae3311ea5ccb:32843 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:1;ae3311ea5ccb:32843 2024-11-15T21:58:58,912 INFO [RS:0;ae3311ea5ccb:35509 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-15T21:58:58,912 INFO [RS:1;ae3311ea5ccb:32843 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-15T21:58:58,912 INFO [RS:1;ae3311ea5ccb:32843 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-15T21:58:58,912 INFO [RS:0;ae3311ea5ccb:35509 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-15T21:58:58,912 DEBUG [RS:1;ae3311ea5ccb:32843 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-15T21:58:58,912 DEBUG [RS:0;ae3311ea5ccb:35509 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-15T21:58:58,916 INFO [RS:0;ae3311ea5ccb:35509 {}] regionserver.HRegionServer(2659): reportForDuty to master=ae3311ea5ccb,40769,1731707936561 with port=35509, startcode=1731707937420 2024-11-15T21:58:58,916 INFO [RS:1;ae3311ea5ccb:32843 {}] regionserver.HRegionServer(2659): reportForDuty to master=ae3311ea5ccb,40769,1731707936561 with port=32843, startcode=1731707937562 2024-11-15T21:58:58,934 DEBUG [RS:0;ae3311ea5ccb:35509 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-15T21:58:58,934 DEBUG [RS:1;ae3311ea5ccb:32843 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-15T21:58:58,948 DEBUG [master/ae3311ea5ccb:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-15T21:58:58,962 INFO [master/ae3311ea5ccb:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2 2024-11-15T21:58:58,975 INFO [master/ae3311ea5ccb:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2024-11-15T21:58:58,982 DEBUG [master/ae3311ea5ccb:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: ae3311ea5ccb,40769,1731707936561 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2024-11-15T21:58:59,001 DEBUG [master/ae3311ea5ccb:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/ae3311ea5ccb:0, corePoolSize=5, maxPoolSize=5 2024-11-15T21:58:59,002 DEBUG [master/ae3311ea5ccb:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/ae3311ea5ccb:0, corePoolSize=5, maxPoolSize=5 2024-11-15T21:58:59,002 DEBUG [master/ae3311ea5ccb:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/ae3311ea5ccb:0, corePoolSize=5, maxPoolSize=5 2024-11-15T21:58:59,002 DEBUG [master/ae3311ea5ccb:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/ae3311ea5ccb:0, corePoolSize=5, maxPoolSize=5 2024-11-15T21:58:59,003 DEBUG [master/ae3311ea5ccb:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/ae3311ea5ccb:0, corePoolSize=10, maxPoolSize=10 2024-11-15T21:58:59,003 DEBUG [master/ae3311ea5ccb:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/ae3311ea5ccb:0, corePoolSize=1, maxPoolSize=1 2024-11-15T21:58:59,003 DEBUG [master/ae3311ea5ccb:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/ae3311ea5ccb:0, corePoolSize=2, maxPoolSize=2 2024-11-15T21:58:59,004 DEBUG [master/ae3311ea5ccb:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/ae3311ea5ccb:0, corePoolSize=1, maxPoolSize=1 2024-11-15T21:58:59,006 INFO [master/ae3311ea5ccb:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731707969006 2024-11-15T21:58:59,008 INFO [master/ae3311ea5ccb:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2024-11-15T21:58:59,009 INFO [master/ae3311ea5ccb:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2024-11-15T21:58:59,012 INFO [master/ae3311ea5ccb:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2024-11-15T21:58:59,013 INFO [master/ae3311ea5ccb:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2024-11-15T21:58:59,013 INFO [master/ae3311ea5ccb:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2024-11-15T21:58:59,013 INFO [master/ae3311ea5ccb:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2024-11-15T21:58:59,015 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-15T21:58:59,016 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region 2024-11-15T21:58:59,020 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T21:58:59,020 INFO [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-11-15T21:58:59,029 INFO [master/ae3311ea5ccb:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-15T21:58:59,034 INFO [master/ae3311ea5ccb:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2024-11-15T21:58:59,035 INFO [master/ae3311ea5ccb:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2024-11-15T21:58:59,036 INFO [master/ae3311ea5ccb:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2024-11-15T21:58:59,040 INFO [master/ae3311ea5ccb:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2024-11-15T21:58:59,040 INFO [master/ae3311ea5ccb:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2024-11-15T21:58:59,044 DEBUG [master/ae3311ea5ccb:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/ae3311ea5ccb:0:becomeActiveMaster-HFileCleaner.large.0-1731707939043,5,FailOnTimeoutGroup] 2024-11-15T21:58:59,044 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:51025, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=RegionServerStatusService 2024-11-15T21:58:59,044 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:34473, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=RegionServerStatusService 2024-11-15T21:58:59,045 DEBUG [master/ae3311ea5ccb:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/ae3311ea5ccb:0:becomeActiveMaster-HFileCleaner.small.0-1731707939044,5,FailOnTimeoutGroup] 2024-11-15T21:58:59,045 INFO [master/ae3311ea5ccb:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-15T21:58:59,046 INFO [master/ae3311ea5ccb:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2024-11-15T21:58:59,047 INFO [master/ae3311ea5ccb:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2024-11-15T21:58:59,047 INFO [master/ae3311ea5ccb:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2024-11-15T21:58:59,050 INFO [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3be1b200-2a9c-651b-3f6b-ac964062b7d6/f4b02001-5f5c-c384-44d5-c547c79a7f1b/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321 2024-11-15T21:58:59,050 INFO [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3be1b200-2a9c-651b-3f6b-ac964062b7d6/f4b02001-5f5c-c384-44d5-c547c79a7f1b 2024-11-15T21:58:59,059 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=40769 {}] master.ServerManager(363): Checking decommissioned status of RegionServer ae3311ea5ccb,35509,1731707937420 2024-11-15T21:58:59,062 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=40769 {}] master.ServerManager(517): Registering regionserver=ae3311ea5ccb,35509,1731707937420 2024-11-15T21:58:59,067 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-15T21:58:59,070 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family info of region 1588230740 2024-11-15T21:58:59,076 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-15T21:58:59,076 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T21:58:59,077 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-15T21:58:59,077 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family ns of region 1588230740 2024-11-15T21:58:59,080 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=40769 {}] master.ServerManager(363): Checking decommissioned status of RegionServer ae3311ea5ccb,32843,1731707937562 2024-11-15T21:58:59,080 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=40769 {}] master.ServerManager(517): Registering regionserver=ae3311ea5ccb,32843,1731707937562 2024-11-15T21:58:59,084 DEBUG [RS:0;ae3311ea5ccb:35509 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3be1b200-2a9c-651b-3f6b-ac964062b7d6/f4b02001-5f5c-c384-44d5-c547c79a7f1b 2024-11-15T21:58:59,085 DEBUG [RS:0;ae3311ea5ccb:35509 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=file:/ 2024-11-15T21:58:59,085 DEBUG [RS:0;ae3311ea5ccb:35509 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-15T21:58:59,085 DEBUG [RS:1;ae3311ea5ccb:32843 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3be1b200-2a9c-651b-3f6b-ac964062b7d6/f4b02001-5f5c-c384-44d5-c547c79a7f1b 2024-11-15T21:58:59,085 DEBUG [RS:1;ae3311ea5ccb:32843 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=file:/ 2024-11-15T21:58:59,085 DEBUG [RS:1;ae3311ea5ccb:32843 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-15T21:58:59,090 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-15T21:58:59,090 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T21:58:59,091 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-15T21:58:59,091 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family rep_barrier of region 1588230740 2024-11-15T21:58:59,095 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40769-0x100f86912970000, quorum=127.0.0.1:62797, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-15T21:58:59,097 DEBUG [RS:1;ae3311ea5ccb:32843 {}] zookeeper.ZKUtil(111): regionserver:32843-0x100f86912970002, quorum=127.0.0.1:62797, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/ae3311ea5ccb,32843,1731707937562 2024-11-15T21:58:59,097 WARN [RS:1;ae3311ea5ccb:32843 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-15T21:58:59,097 DEBUG [RS:0;ae3311ea5ccb:35509 {}] zookeeper.ZKUtil(111): regionserver:35509-0x100f86912970001, quorum=127.0.0.1:62797, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/ae3311ea5ccb,35509,1731707937420 2024-11-15T21:58:59,097 WARN [RS:0;ae3311ea5ccb:35509 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-15T21:58:59,097 INFO [RS:1;ae3311ea5ccb:32843 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-15T21:58:59,097 INFO [RS:0;ae3311ea5ccb:35509 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-15T21:58:59,097 DEBUG [RS:1;ae3311ea5ccb:32843 {}] regionserver.HRegionServer(1793): logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3be1b200-2a9c-651b-3f6b-ac964062b7d6/f4b02001-5f5c-c384-44d5-c547c79a7f1b/WALs/ae3311ea5ccb,32843,1731707937562 2024-11-15T21:58:59,098 DEBUG [RS:0;ae3311ea5ccb:35509 {}] regionserver.HRegionServer(1793): logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3be1b200-2a9c-651b-3f6b-ac964062b7d6/f4b02001-5f5c-c384-44d5-c547c79a7f1b/WALs/ae3311ea5ccb,35509,1731707937420 2024-11-15T21:58:59,103 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-15T21:58:59,103 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T21:58:59,104 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-15T21:58:59,104 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family table of region 1588230740 2024-11-15T21:58:59,106 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [ae3311ea5ccb,32843,1731707937562] 2024-11-15T21:58:59,106 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [ae3311ea5ccb,35509,1731707937420] 2024-11-15T21:58:59,115 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-15T21:58:59,117 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T21:58:59,117 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-15T21:58:59,118 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-15T21:58:59,119 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3be1b200-2a9c-651b-3f6b-ac964062b7d6/f4b02001-5f5c-c384-44d5-c547c79a7f1b/data/hbase/meta/1588230740 2024-11-15T21:58:59,119 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3be1b200-2a9c-651b-3f6b-ac964062b7d6/f4b02001-5f5c-c384-44d5-c547c79a7f1b/data/hbase/meta/1588230740 2024-11-15T21:58:59,120 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-15T21:58:59,120 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-15T21:58:59,120 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-15T21:58:59,121 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-15T21:58:59,132 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3be1b200-2a9c-651b-3f6b-ac964062b7d6/f4b02001-5f5c-c384-44d5-c547c79a7f1b/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-15T21:58:59,132 INFO [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=64464654, jitterRate=-0.03940179944038391}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-15T21:58:59,136 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740: Writing region info on filesystem at 1731707939069Initializing all the Stores at 1731707939070 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731707939070Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731707939070Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731707939070Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731707939070Cleaning up temporary data from old regions at 1731707939120 (+50 ms)Region opened successfully at 1731707939136 (+16 ms) 2024-11-15T21:58:59,136 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-15T21:58:59,137 INFO [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-15T21:58:59,137 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-15T21:58:59,137 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-15T21:58:59,137 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-15T21:58:59,139 INFO [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-15T21:58:59,139 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1731707939136Disabling compacts and flushes for region at 1731707939136Disabling writes for close at 1731707939137 (+1 ms)Writing region close event to WAL at 1731707939138 (+1 ms)Closed at 1731707939139 (+1 ms) 2024-11-15T21:58:59,141 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-15T21:58:59,141 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta 2024-11-15T21:58:59,150 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-11-15T21:58:59,158 INFO [RS:1;ae3311ea5ccb:32843 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-15T21:58:59,158 INFO [RS:0;ae3311ea5ccb:35509 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-15T21:58:59,159 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-15T21:58:59,163 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2024-11-15T21:58:59,175 INFO [RS:1;ae3311ea5ccb:32843 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-15T21:58:59,177 INFO [RS:0;ae3311ea5ccb:35509 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-15T21:58:59,181 INFO [RS:0;ae3311ea5ccb:35509 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-15T21:58:59,181 INFO [RS:1;ae3311ea5ccb:32843 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-15T21:58:59,181 INFO [RS:1;ae3311ea5ccb:32843 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-15T21:58:59,181 INFO [RS:0;ae3311ea5ccb:35509 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-15T21:58:59,186 INFO [RS:1;ae3311ea5ccb:32843 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-15T21:58:59,186 INFO [RS:0;ae3311ea5ccb:35509 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-15T21:58:59,191 INFO [RS:0;ae3311ea5ccb:35509 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-15T21:58:59,191 INFO [RS:1;ae3311ea5ccb:32843 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-15T21:58:59,193 INFO [RS:0;ae3311ea5ccb:35509 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-15T21:58:59,193 DEBUG [RS:0;ae3311ea5ccb:35509 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/ae3311ea5ccb:0, corePoolSize=1, maxPoolSize=1 2024-11-15T21:58:59,193 DEBUG [RS:0;ae3311ea5ccb:35509 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/ae3311ea5ccb:0, corePoolSize=1, maxPoolSize=1 2024-11-15T21:58:59,194 DEBUG [RS:0;ae3311ea5ccb:35509 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/ae3311ea5ccb:0, corePoolSize=1, maxPoolSize=1 2024-11-15T21:58:59,194 DEBUG [RS:0;ae3311ea5ccb:35509 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/ae3311ea5ccb:0, corePoolSize=1, maxPoolSize=1 2024-11-15T21:58:59,194 DEBUG [RS:0;ae3311ea5ccb:35509 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/ae3311ea5ccb:0, corePoolSize=1, maxPoolSize=1 2024-11-15T21:58:59,194 DEBUG [RS:0;ae3311ea5ccb:35509 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/ae3311ea5ccb:0, corePoolSize=2, maxPoolSize=2 2024-11-15T21:58:59,194 DEBUG [RS:0;ae3311ea5ccb:35509 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/ae3311ea5ccb:0, corePoolSize=1, maxPoolSize=1 2024-11-15T21:58:59,195 INFO [RS:1;ae3311ea5ccb:32843 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-15T21:58:59,195 DEBUG [RS:0;ae3311ea5ccb:35509 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/ae3311ea5ccb:0, corePoolSize=1, maxPoolSize=1 2024-11-15T21:58:59,195 DEBUG [RS:1;ae3311ea5ccb:32843 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/ae3311ea5ccb:0, corePoolSize=1, maxPoolSize=1 2024-11-15T21:58:59,195 DEBUG [RS:0;ae3311ea5ccb:35509 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/ae3311ea5ccb:0, corePoolSize=1, maxPoolSize=1 2024-11-15T21:58:59,195 DEBUG [RS:1;ae3311ea5ccb:32843 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/ae3311ea5ccb:0, corePoolSize=1, maxPoolSize=1 2024-11-15T21:58:59,195 DEBUG [RS:0;ae3311ea5ccb:35509 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/ae3311ea5ccb:0, corePoolSize=1, maxPoolSize=1 2024-11-15T21:58:59,195 DEBUG [RS:1;ae3311ea5ccb:32843 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/ae3311ea5ccb:0, corePoolSize=1, maxPoolSize=1 2024-11-15T21:58:59,195 DEBUG [RS:0;ae3311ea5ccb:35509 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/ae3311ea5ccb:0, corePoolSize=1, maxPoolSize=1 2024-11-15T21:58:59,196 DEBUG [RS:1;ae3311ea5ccb:32843 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/ae3311ea5ccb:0, corePoolSize=1, maxPoolSize=1 2024-11-15T21:58:59,196 DEBUG [RS:0;ae3311ea5ccb:35509 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/ae3311ea5ccb:0, corePoolSize=1, maxPoolSize=1 2024-11-15T21:58:59,196 DEBUG [RS:0;ae3311ea5ccb:35509 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/ae3311ea5ccb:0, corePoolSize=3, maxPoolSize=3 2024-11-15T21:58:59,196 DEBUG [RS:1;ae3311ea5ccb:32843 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/ae3311ea5ccb:0, corePoolSize=1, maxPoolSize=1 2024-11-15T21:58:59,196 DEBUG [RS:1;ae3311ea5ccb:32843 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/ae3311ea5ccb:0, corePoolSize=2, maxPoolSize=2 2024-11-15T21:58:59,196 DEBUG [RS:0;ae3311ea5ccb:35509 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0, corePoolSize=3, maxPoolSize=3 2024-11-15T21:58:59,196 DEBUG [RS:1;ae3311ea5ccb:32843 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/ae3311ea5ccb:0, corePoolSize=1, maxPoolSize=1 2024-11-15T21:58:59,196 DEBUG [RS:1;ae3311ea5ccb:32843 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/ae3311ea5ccb:0, corePoolSize=1, maxPoolSize=1 2024-11-15T21:58:59,197 DEBUG [RS:1;ae3311ea5ccb:32843 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/ae3311ea5ccb:0, corePoolSize=1, maxPoolSize=1 2024-11-15T21:58:59,197 DEBUG [RS:1;ae3311ea5ccb:32843 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/ae3311ea5ccb:0, corePoolSize=1, maxPoolSize=1 2024-11-15T21:58:59,197 DEBUG [RS:1;ae3311ea5ccb:32843 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/ae3311ea5ccb:0, corePoolSize=1, maxPoolSize=1 2024-11-15T21:58:59,197 DEBUG [RS:1;ae3311ea5ccb:32843 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/ae3311ea5ccb:0, corePoolSize=1, maxPoolSize=1 2024-11-15T21:58:59,197 DEBUG [RS:1;ae3311ea5ccb:32843 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/ae3311ea5ccb:0, corePoolSize=3, maxPoolSize=3 2024-11-15T21:58:59,198 DEBUG [RS:1;ae3311ea5ccb:32843 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0, corePoolSize=3, maxPoolSize=3 2024-11-15T21:58:59,209 INFO [RS:0;ae3311ea5ccb:35509 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-15T21:58:59,210 INFO [RS:0;ae3311ea5ccb:35509 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-15T21:58:59,210 INFO [RS:0;ae3311ea5ccb:35509 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-15T21:58:59,210 INFO [RS:0;ae3311ea5ccb:35509 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-15T21:58:59,210 INFO [RS:0;ae3311ea5ccb:35509 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-15T21:58:59,210 INFO [RS:0;ae3311ea5ccb:35509 {}] hbase.ChoreService(168): Chore ScheduledChore name=ae3311ea5ccb,35509,1731707937420-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-15T21:58:59,211 INFO [RS:1;ae3311ea5ccb:32843 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-15T21:58:59,211 INFO [RS:1;ae3311ea5ccb:32843 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-15T21:58:59,211 INFO [RS:1;ae3311ea5ccb:32843 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-15T21:58:59,211 INFO [RS:1;ae3311ea5ccb:32843 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-15T21:58:59,211 INFO [RS:1;ae3311ea5ccb:32843 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-15T21:58:59,211 INFO [RS:1;ae3311ea5ccb:32843 {}] hbase.ChoreService(168): Chore ScheduledChore name=ae3311ea5ccb,32843,1731707937562-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-15T21:58:59,236 INFO [RS:0;ae3311ea5ccb:35509 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-15T21:58:59,237 INFO [RS:1;ae3311ea5ccb:32843 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-15T21:58:59,239 INFO [RS:0;ae3311ea5ccb:35509 {}] hbase.ChoreService(168): Chore ScheduledChore name=ae3311ea5ccb,35509,1731707937420-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-15T21:58:59,239 INFO [RS:0;ae3311ea5ccb:35509 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-15T21:58:59,239 INFO [RS:0;ae3311ea5ccb:35509 {}] regionserver.Replication(171): ae3311ea5ccb,35509,1731707937420 started 2024-11-15T21:58:59,245 INFO [RS:1;ae3311ea5ccb:32843 {}] hbase.ChoreService(168): Chore ScheduledChore name=ae3311ea5ccb,32843,1731707937562-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-15T21:58:59,246 INFO [RS:1;ae3311ea5ccb:32843 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-15T21:58:59,246 INFO [RS:1;ae3311ea5ccb:32843 {}] regionserver.Replication(171): ae3311ea5ccb,32843,1731707937562 started 2024-11-15T21:58:59,268 INFO [RS:0;ae3311ea5ccb:35509 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-15T21:58:59,269 INFO [RS:0;ae3311ea5ccb:35509 {}] regionserver.HRegionServer(1482): Serving as ae3311ea5ccb,35509,1731707937420, RpcServer on ae3311ea5ccb/172.17.0.3:35509, sessionid=0x100f86912970001 2024-11-15T21:58:59,270 DEBUG [RS:0;ae3311ea5ccb:35509 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-15T21:58:59,270 DEBUG [RS:0;ae3311ea5ccb:35509 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager ae3311ea5ccb,35509,1731707937420 2024-11-15T21:58:59,270 DEBUG [RS:0;ae3311ea5ccb:35509 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'ae3311ea5ccb,35509,1731707937420' 2024-11-15T21:58:59,271 DEBUG [RS:0;ae3311ea5ccb:35509 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-15T21:58:59,272 INFO [RS:1;ae3311ea5ccb:32843 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-15T21:58:59,272 INFO [RS:1;ae3311ea5ccb:32843 {}] regionserver.HRegionServer(1482): Serving as ae3311ea5ccb,32843,1731707937562, RpcServer on ae3311ea5ccb/172.17.0.3:32843, sessionid=0x100f86912970002 2024-11-15T21:58:59,272 DEBUG [RS:1;ae3311ea5ccb:32843 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-15T21:58:59,272 DEBUG [RS:1;ae3311ea5ccb:32843 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager ae3311ea5ccb,32843,1731707937562 2024-11-15T21:58:59,272 DEBUG [RS:1;ae3311ea5ccb:32843 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'ae3311ea5ccb,32843,1731707937562' 2024-11-15T21:58:59,272 DEBUG [RS:1;ae3311ea5ccb:32843 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-15T21:58:59,274 DEBUG [RS:0;ae3311ea5ccb:35509 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-15T21:58:59,274 DEBUG [RS:1;ae3311ea5ccb:32843 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-15T21:58:59,275 DEBUG [RS:0;ae3311ea5ccb:35509 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-15T21:58:59,275 DEBUG [RS:0;ae3311ea5ccb:35509 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-15T21:58:59,275 DEBUG [RS:1;ae3311ea5ccb:32843 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-15T21:58:59,275 DEBUG [RS:0;ae3311ea5ccb:35509 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager ae3311ea5ccb,35509,1731707937420 2024-11-15T21:58:59,275 DEBUG [RS:1;ae3311ea5ccb:32843 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-15T21:58:59,275 DEBUG [RS:0;ae3311ea5ccb:35509 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'ae3311ea5ccb,35509,1731707937420' 2024-11-15T21:58:59,275 DEBUG [RS:1;ae3311ea5ccb:32843 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager ae3311ea5ccb,32843,1731707937562 2024-11-15T21:58:59,275 DEBUG [RS:0;ae3311ea5ccb:35509 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-15T21:58:59,275 DEBUG [RS:1;ae3311ea5ccb:32843 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'ae3311ea5ccb,32843,1731707937562' 2024-11-15T21:58:59,275 DEBUG [RS:1;ae3311ea5ccb:32843 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-15T21:58:59,276 DEBUG [RS:0;ae3311ea5ccb:35509 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-15T21:58:59,276 DEBUG [RS:1;ae3311ea5ccb:32843 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-15T21:58:59,277 DEBUG [RS:0;ae3311ea5ccb:35509 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-15T21:58:59,277 INFO [RS:0;ae3311ea5ccb:35509 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-15T21:58:59,277 INFO [RS:0;ae3311ea5ccb:35509 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-15T21:58:59,277 DEBUG [RS:1;ae3311ea5ccb:32843 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-15T21:58:59,278 INFO [RS:1;ae3311ea5ccb:32843 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-15T21:58:59,278 INFO [RS:1;ae3311ea5ccb:32843 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-15T21:58:59,314 WARN [ae3311ea5ccb:40769 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-15T21:58:59,384 INFO [RS:1;ae3311ea5ccb:32843 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-15T21:58:59,384 INFO [RS:0;ae3311ea5ccb:35509 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-15T21:58:59,385 INFO [RS:1;ae3311ea5ccb:32843 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=ae3311ea5ccb%2C32843%2C1731707937562, suffix=, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3be1b200-2a9c-651b-3f6b-ac964062b7d6/f4b02001-5f5c-c384-44d5-c547c79a7f1b/WALs/ae3311ea5ccb,32843,1731707937562, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3be1b200-2a9c-651b-3f6b-ac964062b7d6/f4b02001-5f5c-c384-44d5-c547c79a7f1b/oldWALs, maxLogs=55 2024-11-15T21:58:59,385 INFO [RS:0;ae3311ea5ccb:35509 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=ae3311ea5ccb%2C35509%2C1731707937420, suffix=, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3be1b200-2a9c-651b-3f6b-ac964062b7d6/f4b02001-5f5c-c384-44d5-c547c79a7f1b/WALs/ae3311ea5ccb,35509,1731707937420, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3be1b200-2a9c-651b-3f6b-ac964062b7d6/f4b02001-5f5c-c384-44d5-c547c79a7f1b/oldWALs, maxLogs=55 2024-11-15T21:58:59,413 INFO [RS:1;ae3311ea5ccb:32843 {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3be1b200-2a9c-651b-3f6b-ac964062b7d6/f4b02001-5f5c-c384-44d5-c547c79a7f1b/WALs/ae3311ea5ccb,32843,1731707937562/ae3311ea5ccb%2C32843%2C1731707937562.1731707939390 2024-11-15T21:58:59,414 DEBUG [RS:1;ae3311ea5ccb:32843 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-11-15T21:58:59,414 INFO [RS:0;ae3311ea5ccb:35509 {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3be1b200-2a9c-651b-3f6b-ac964062b7d6/f4b02001-5f5c-c384-44d5-c547c79a7f1b/WALs/ae3311ea5ccb,35509,1731707937420/ae3311ea5ccb%2C35509%2C1731707937420.1731707939390 2024-11-15T21:58:59,414 DEBUG [RS:0;ae3311ea5ccb:35509 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-11-15T21:58:59,566 DEBUG [ae3311ea5ccb:40769 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=2, allServersCount=2 2024-11-15T21:58:59,578 DEBUG [ae3311ea5ccb:40769 {}] balancer.BalancerClusterState(204): Hosts are {ae3311ea5ccb=0} racks are {/default-rack=0} 2024-11-15T21:58:59,581 DEBUG [ae3311ea5ccb:40769 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-11-15T21:58:59,582 DEBUG [ae3311ea5ccb:40769 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-11-15T21:58:59,582 DEBUG [ae3311ea5ccb:40769 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-11-15T21:58:59,582 DEBUG [ae3311ea5ccb:40769 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-11-15T21:58:59,582 INFO [ae3311ea5ccb:40769 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-11-15T21:58:59,582 INFO [ae3311ea5ccb:40769 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-11-15T21:58:59,582 DEBUG [ae3311ea5ccb:40769 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-11-15T21:58:59,589 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=ae3311ea5ccb,32843,1731707937562 2024-11-15T21:58:59,595 INFO [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as ae3311ea5ccb,32843,1731707937562, state=OPENING 2024-11-15T21:58:59,602 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2024-11-15T21:58:59,605 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:32843-0x100f86912970002, quorum=127.0.0.1:62797, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-15T21:58:59,605 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40769-0x100f86912970000, quorum=127.0.0.1:62797, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-15T21:58:59,605 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35509-0x100f86912970001, quorum=127.0.0.1:62797, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-15T21:58:59,607 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-15T21:58:59,607 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-15T21:58:59,607 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-15T21:58:59,608 DEBUG [PEWorker-3 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-15T21:58:59,610 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=ae3311ea5ccb,32843,1731707937562}] 2024-11-15T21:58:59,784 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-15T21:58:59,788 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-4-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:59251, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-15T21:58:59,796 INFO [RS_OPEN_META-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-11-15T21:58:59,796 INFO [RS_OPEN_META-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-15T21:58:59,797 INFO [RS_OPEN_META-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-11-15T21:58:59,798 INFO [RS_OPEN_META-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=ae3311ea5ccb%2C32843%2C1731707937562.meta, suffix=.meta, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3be1b200-2a9c-651b-3f6b-ac964062b7d6/f4b02001-5f5c-c384-44d5-c547c79a7f1b/WALs/ae3311ea5ccb,32843,1731707937562, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3be1b200-2a9c-651b-3f6b-ac964062b7d6/f4b02001-5f5c-c384-44d5-c547c79a7f1b/oldWALs, maxLogs=55 2024-11-15T21:58:59,819 INFO [RS_OPEN_META-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3be1b200-2a9c-651b-3f6b-ac964062b7d6/f4b02001-5f5c-c384-44d5-c547c79a7f1b/WALs/ae3311ea5ccb,32843,1731707937562/ae3311ea5ccb%2C32843%2C1731707937562.meta.1731707939800.meta 2024-11-15T21:58:59,820 DEBUG [RS_OPEN_META-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-11-15T21:58:59,820 DEBUG [RS_OPEN_META-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-11-15T21:58:59,822 DEBUG [RS_OPEN_META-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-11-15T21:58:59,825 DEBUG [RS_OPEN_META-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-11-15T21:58:59,831 INFO [RS_OPEN_META-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-11-15T21:58:59,835 DEBUG [RS_OPEN_META-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-11-15T21:58:59,836 DEBUG [RS_OPEN_META-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-15T21:58:59,836 DEBUG [RS_OPEN_META-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-11-15T21:58:59,836 DEBUG [RS_OPEN_META-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-11-15T21:58:59,838 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family info of region 1588230740 2024-11-15T21:58:59,839 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-15T21:58:59,840 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T21:58:59,840 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-15T21:58:59,840 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family ns of region 1588230740 2024-11-15T21:58:59,841 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-15T21:58:59,842 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T21:58:59,842 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-15T21:58:59,842 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family rep_barrier of region 1588230740 2024-11-15T21:58:59,843 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-15T21:58:59,844 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T21:58:59,844 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-15T21:58:59,844 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family table of region 1588230740 2024-11-15T21:58:59,845 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-15T21:58:59,845 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T21:58:59,846 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-15T21:58:59,846 DEBUG [RS_OPEN_META-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-15T21:58:59,847 DEBUG [RS_OPEN_META-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3be1b200-2a9c-651b-3f6b-ac964062b7d6/f4b02001-5f5c-c384-44d5-c547c79a7f1b/data/hbase/meta/1588230740 2024-11-15T21:58:59,848 DEBUG [RS_OPEN_META-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3be1b200-2a9c-651b-3f6b-ac964062b7d6/f4b02001-5f5c-c384-44d5-c547c79a7f1b/data/hbase/meta/1588230740 2024-11-15T21:58:59,848 DEBUG [RS_OPEN_META-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-15T21:58:59,848 DEBUG [RS_OPEN_META-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-15T21:58:59,849 DEBUG [RS_OPEN_META-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-15T21:58:59,850 DEBUG [RS_OPEN_META-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-15T21:58:59,851 INFO [RS_OPEN_META-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=68052935, jitterRate=0.01406775414943695}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-15T21:58:59,851 DEBUG [RS_OPEN_META-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-11-15T21:58:59,853 DEBUG [RS_OPEN_META-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1731707939837Writing region info on filesystem at 1731707939837Initializing all the Stores at 1731707939837Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731707939838 (+1 ms)Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731707939838Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731707939838Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731707939838Cleaning up temporary data from old regions at 1731707939848 (+10 ms)Running coprocessor post-open hooks at 1731707939851 (+3 ms)Region opened successfully at 1731707939853 (+2 ms) 2024-11-15T21:58:59,859 INFO [RS_OPEN_META-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1731707939775 2024-11-15T21:58:59,870 DEBUG [RS_OPEN_META-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-11-15T21:58:59,870 INFO [RS_OPEN_META-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-11-15T21:58:59,871 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=ae3311ea5ccb,32843,1731707937562 2024-11-15T21:58:59,873 INFO [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as ae3311ea5ccb,32843,1731707937562, state=OPEN 2024-11-15T21:58:59,876 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:32843-0x100f86912970002, quorum=127.0.0.1:62797, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-15T21:58:59,876 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40769-0x100f86912970000, quorum=127.0.0.1:62797, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-15T21:58:59,875 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35509-0x100f86912970001, quorum=127.0.0.1:62797, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-15T21:58:59,876 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-15T21:58:59,876 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-15T21:58:59,876 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-15T21:58:59,876 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=ae3311ea5ccb,32843,1731707937562 2024-11-15T21:58:59,879 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2 2024-11-15T21:58:59,880 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=ae3311ea5ccb,32843,1731707937562 in 267 msec 2024-11-15T21:58:59,883 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1 2024-11-15T21:58:59,884 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 731 msec 2024-11-15T21:58:59,884 DEBUG [PEWorker-2 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-15T21:58:59,884 INFO [PEWorker-2 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces 2024-11-15T21:58:59,905 DEBUG [PEWorker-2 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-15T21:58:59,907 DEBUG [PEWorker-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=ae3311ea5ccb,32843,1731707937562, seqNum=-1] 2024-11-15T21:58:59,928 DEBUG [PEWorker-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-15T21:58:59,930 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-4-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:55033, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-15T21:58:59,948 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 1.0630 sec 2024-11-15T21:58:59,948 INFO [master/ae3311ea5ccb:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1731707939948, completionTime=-1 2024-11-15T21:58:59,951 INFO [master/ae3311ea5ccb:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=2; waited=0ms, expected min=2 server(s), max=2 server(s), master is running 2024-11-15T21:58:59,951 DEBUG [master/ae3311ea5ccb:0:becomeActiveMaster {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-15T21:58:59,981 INFO [master/ae3311ea5ccb:0:becomeActiveMaster {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-11-15T21:58:59,981 INFO [master/ae3311ea5ccb:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731707999981 2024-11-15T21:58:59,981 INFO [master/ae3311ea5ccb:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731708059981 2024-11-15T21:58:59,981 INFO [master/ae3311ea5ccb:0:becomeActiveMaster {}] assignment.AssignmentManager(1783): Joined the cluster in 30 msec 2024-11-15T21:58:59,984 DEBUG [master/ae3311ea5ccb:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 1588230740 changed from -1.0 to 0.0, refreshing cache 2024-11-15T21:58:59,992 INFO [master/ae3311ea5ccb:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ae3311ea5ccb,40769,1731707936561-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-15T21:58:59,992 INFO [master/ae3311ea5ccb:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ae3311ea5ccb,40769,1731707936561-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-15T21:58:59,992 INFO [master/ae3311ea5ccb:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ae3311ea5ccb,40769,1731707936561-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-15T21:58:59,994 INFO [master/ae3311ea5ccb:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-ae3311ea5ccb:40769, period=300000, unit=MILLISECONDS is enabled. 2024-11-15T21:58:59,995 INFO [master/ae3311ea5ccb:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2024-11-15T21:58:59,995 INFO [master/ae3311ea5ccb:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled. 2024-11-15T21:59:00,001 DEBUG [master/ae3311ea5ccb:0.Chore.1 {}] janitor.CatalogJanitor(180): 2024-11-15T21:59:00,026 INFO [master/ae3311ea5ccb:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 2.298sec 2024-11-15T21:59:00,028 INFO [master/ae3311ea5ccb:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2024-11-15T21:59:00,030 INFO [master/ae3311ea5ccb:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2024-11-15T21:59:00,031 INFO [master/ae3311ea5ccb:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2024-11-15T21:59:00,031 INFO [master/ae3311ea5ccb:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2024-11-15T21:59:00,031 INFO [master/ae3311ea5ccb:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2024-11-15T21:59:00,032 INFO [master/ae3311ea5ccb:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ae3311ea5ccb,40769,1731707936561-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-15T21:59:00,032 INFO [master/ae3311ea5ccb:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ae3311ea5ccb,40769,1731707936561-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2024-11-15T21:59:00,034 DEBUG [master/ae3311ea5ccb:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds 2024-11-15T21:59:00,034 INFO [master/ae3311ea5ccb:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2024-11-15T21:59:00,035 INFO [master/ae3311ea5ccb:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ae3311ea5ccb,40769,1731707936561-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-15T21:59:00,071 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@32be4008, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T21:59:00,073 DEBUG [Time-limited test {}] nio.NioEventLoop(110): -Dio.netty.noKeySetOptimization: false 2024-11-15T21:59:00,073 DEBUG [Time-limited test {}] nio.NioEventLoop(111): -Dio.netty.selectorAutoRebuildThreshold: 512 2024-11-15T21:59:00,077 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request ae3311ea5ccb,40769,-1 for getting cluster id 2024-11-15T21:59:00,080 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-15T21:59:00,088 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '02d33e22-0b9b-43a7-a826-fb44bc49b2d6' 2024-11-15T21:59:00,090 DEBUG [RPCClient-NioEventLoopGroup-5-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-15T21:59:00,090 DEBUG [RPCClient-NioEventLoopGroup-5-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "02d33e22-0b9b-43a7-a826-fb44bc49b2d6" 2024-11-15T21:59:00,093 DEBUG [RPCClient-NioEventLoopGroup-5-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@433c3136, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T21:59:00,093 DEBUG [RPCClient-NioEventLoopGroup-5-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [ae3311ea5ccb,40769,-1] 2024-11-15T21:59:00,095 DEBUG [RPCClient-NioEventLoopGroup-5-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-15T21:59:00,097 DEBUG [RPCClient-NioEventLoopGroup-5-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T21:59:00,099 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:60390, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-15T21:59:00,102 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3302ae21, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T21:59:00,103 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-15T21:59:00,111 DEBUG [RPCClient-NioEventLoopGroup-5-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=ae3311ea5ccb,32843,1731707937562, seqNum=-1] 2024-11-15T21:59:00,111 DEBUG [RPCClient-NioEventLoopGroup-5-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-15T21:59:00,114 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-4-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:48586, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-15T21:59:00,151 INFO [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=ae3311ea5ccb,40769,1731707936561 2024-11-15T21:59:00,158 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-15T21:59:00,172 DEBUG [RPCClient-NioEventLoopGroup-5-2 {}] client.AsyncConnectionImpl(321): The fetched master address is ae3311ea5ccb,40769,1731707936561 2024-11-15T21:59:00,175 DEBUG [RPCClient-NioEventLoopGroup-5-2 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@1e208b87 2024-11-15T21:59:00,177 DEBUG [RPCClient-NioEventLoopGroup-5-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-15T21:59:00,180 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:60400, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-15T21:59:00,186 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40769 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.3 create 'testBlockEvictionOnRegionMove1731707940151', {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-15T21:59:00,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40769 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=testBlockEvictionOnRegionMove1731707940151 2024-11-15T21:59:00,196 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=testBlockEvictionOnRegionMove1731707940151 execute state=CREATE_TABLE_PRE_OPERATION 2024-11-15T21:59:00,198 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40769 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.3 procedure request for creating table: namespace: "default" qualifier: "testBlockEvictionOnRegionMove1731707940151" procId is: 4 2024-11-15T21:59:00,199 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T21:59:00,200 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=testBlockEvictionOnRegionMove1731707940151 execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-15T21:59:00,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40769 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-15T21:59:00,216 INFO [RegionOpenAndInit-testBlockEvictionOnRegionMove1731707940151-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => b70e50bf67275a89944d13e9626080b2, NAME => 'testBlockEvictionOnRegionMove1731707940151,,1731707940181.b70e50bf67275a89944d13e9626080b2.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testBlockEvictionOnRegionMove1731707940151', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3be1b200-2a9c-651b-3f6b-ac964062b7d6/f4b02001-5f5c-c384-44d5-c547c79a7f1b 2024-11-15T21:59:00,223 DEBUG [RegionOpenAndInit-testBlockEvictionOnRegionMove1731707940151-pool-0 {}] regionserver.HRegion(898): Instantiated testBlockEvictionOnRegionMove1731707940151,,1731707940181.b70e50bf67275a89944d13e9626080b2.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-15T21:59:00,224 DEBUG [RegionOpenAndInit-testBlockEvictionOnRegionMove1731707940151-pool-0 {}] regionserver.HRegion(1722): Closing b70e50bf67275a89944d13e9626080b2, disabling compactions & flushes 2024-11-15T21:59:00,224 INFO [RegionOpenAndInit-testBlockEvictionOnRegionMove1731707940151-pool-0 {}] regionserver.HRegion(1755): Closing region testBlockEvictionOnRegionMove1731707940151,,1731707940181.b70e50bf67275a89944d13e9626080b2. 2024-11-15T21:59:00,224 DEBUG [RegionOpenAndInit-testBlockEvictionOnRegionMove1731707940151-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on testBlockEvictionOnRegionMove1731707940151,,1731707940181.b70e50bf67275a89944d13e9626080b2. 2024-11-15T21:59:00,224 DEBUG [RegionOpenAndInit-testBlockEvictionOnRegionMove1731707940151-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on testBlockEvictionOnRegionMove1731707940151,,1731707940181.b70e50bf67275a89944d13e9626080b2. after waiting 0 ms 2024-11-15T21:59:00,224 DEBUG [RegionOpenAndInit-testBlockEvictionOnRegionMove1731707940151-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region testBlockEvictionOnRegionMove1731707940151,,1731707940181.b70e50bf67275a89944d13e9626080b2. 2024-11-15T21:59:00,224 INFO [RegionOpenAndInit-testBlockEvictionOnRegionMove1731707940151-pool-0 {}] regionserver.HRegion(1973): Closed testBlockEvictionOnRegionMove1731707940151,,1731707940181.b70e50bf67275a89944d13e9626080b2. 2024-11-15T21:59:00,224 DEBUG [RegionOpenAndInit-testBlockEvictionOnRegionMove1731707940151-pool-0 {}] regionserver.HRegion(1676): Region close journal for b70e50bf67275a89944d13e9626080b2: Waiting for close lock at 1731707940223Disabling compacts and flushes for region at 1731707940223Disabling writes for close at 1731707940224 (+1 ms)Writing region close event to WAL at 1731707940224Closed at 1731707940224 2024-11-15T21:59:00,226 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=testBlockEvictionOnRegionMove1731707940151 execute state=CREATE_TABLE_ADD_TO_META 2024-11-15T21:59:00,231 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testBlockEvictionOnRegionMove1731707940151,,1731707940181.b70e50bf67275a89944d13e9626080b2.","families":{"info":[{"qualifier":"regioninfo","vlen":76,"tag":[],"timestamp":"1731707940226"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731707940226"}]},"ts":"1731707940226"} 2024-11-15T21:59:00,236 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-15T21:59:00,237 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=testBlockEvictionOnRegionMove1731707940151 execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-15T21:59:00,240 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testBlockEvictionOnRegionMove1731707940151","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731707940237"}]},"ts":"1731707940237"} 2024-11-15T21:59:00,244 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=testBlockEvictionOnRegionMove1731707940151, state=ENABLING in hbase:meta 2024-11-15T21:59:00,245 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(204): Hosts are {ae3311ea5ccb=0} racks are {/default-rack=0} 2024-11-15T21:59:00,246 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-11-15T21:59:00,246 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-11-15T21:59:00,246 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-11-15T21:59:00,246 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-11-15T21:59:00,246 INFO [PEWorker-3 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-11-15T21:59:00,246 INFO [PEWorker-3 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-11-15T21:59:00,246 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-11-15T21:59:00,248 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1731707940151, region=b70e50bf67275a89944d13e9626080b2, ASSIGN}] 2024-11-15T21:59:00,250 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1731707940151, region=b70e50bf67275a89944d13e9626080b2, ASSIGN 2024-11-15T21:59:00,251 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1731707940151, region=b70e50bf67275a89944d13e9626080b2, ASSIGN; state=OFFLINE, location=ae3311ea5ccb,35509,1731707937420; forceNewPlan=false, retain=false 2024-11-15T21:59:00,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40769 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-15T21:59:00,404 INFO [ae3311ea5ccb:40769 {}] balancer.BaseLoadBalancer(388): Reassigned 1 regions. 1 retained the pre-restart assignment. 2024-11-15T21:59:00,405 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=b70e50bf67275a89944d13e9626080b2, regionState=OPENING, regionLocation=ae3311ea5ccb,35509,1731707937420 2024-11-15T21:59:00,409 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-4-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1731707940151, region=b70e50bf67275a89944d13e9626080b2, ASSIGN because future has completed 2024-11-15T21:59:00,410 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure b70e50bf67275a89944d13e9626080b2, server=ae3311ea5ccb,35509,1731707937420}] 2024-11-15T21:59:00,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40769 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-15T21:59:00,564 DEBUG [RSProcedureDispatcher-pool-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-15T21:59:00,571 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:40341, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-15T21:59:00,574 INFO [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(132): Open testBlockEvictionOnRegionMove1731707940151,,1731707940181.b70e50bf67275a89944d13e9626080b2. 2024-11-15T21:59:00,574 DEBUG [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7752): Opening region: {ENCODED => b70e50bf67275a89944d13e9626080b2, NAME => 'testBlockEvictionOnRegionMove1731707940151,,1731707940181.b70e50bf67275a89944d13e9626080b2.', STARTKEY => '', ENDKEY => ''} 2024-11-15T21:59:00,575 DEBUG [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testBlockEvictionOnRegionMove1731707940151 b70e50bf67275a89944d13e9626080b2 2024-11-15T21:59:00,575 DEBUG [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(898): Instantiated testBlockEvictionOnRegionMove1731707940151,,1731707940181.b70e50bf67275a89944d13e9626080b2.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-15T21:59:00,575 DEBUG [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7794): checking encryption for b70e50bf67275a89944d13e9626080b2 2024-11-15T21:59:00,576 DEBUG [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7797): checking classloading for b70e50bf67275a89944d13e9626080b2 2024-11-15T21:59:00,581 INFO [StoreOpener-b70e50bf67275a89944d13e9626080b2-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family family of region b70e50bf67275a89944d13e9626080b2 2024-11-15T21:59:00,590 INFO [StoreOpener-b70e50bf67275a89944d13e9626080b2-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region b70e50bf67275a89944d13e9626080b2 columnFamilyName family 2024-11-15T21:59:00,590 DEBUG [StoreOpener-b70e50bf67275a89944d13e9626080b2-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T21:59:00,591 INFO [StoreOpener-b70e50bf67275a89944d13e9626080b2-1 {}] regionserver.HStore(327): Store=b70e50bf67275a89944d13e9626080b2/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-15T21:59:00,591 DEBUG [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1038): replaying wal for b70e50bf67275a89944d13e9626080b2 2024-11-15T21:59:00,593 DEBUG [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3be1b200-2a9c-651b-3f6b-ac964062b7d6/f4b02001-5f5c-c384-44d5-c547c79a7f1b/data/default/testBlockEvictionOnRegionMove1731707940151/b70e50bf67275a89944d13e9626080b2 2024-11-15T21:59:00,593 DEBUG [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3be1b200-2a9c-651b-3f6b-ac964062b7d6/f4b02001-5f5c-c384-44d5-c547c79a7f1b/data/default/testBlockEvictionOnRegionMove1731707940151/b70e50bf67275a89944d13e9626080b2 2024-11-15T21:59:00,593 DEBUG [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1048): stopping wal replay for b70e50bf67275a89944d13e9626080b2 2024-11-15T21:59:00,593 DEBUG [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1060): Cleaning up temporary data for b70e50bf67275a89944d13e9626080b2 2024-11-15T21:59:00,595 DEBUG [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1093): writing seq id for b70e50bf67275a89944d13e9626080b2 2024-11-15T21:59:00,620 DEBUG [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3be1b200-2a9c-651b-3f6b-ac964062b7d6/f4b02001-5f5c-c384-44d5-c547c79a7f1b/data/default/testBlockEvictionOnRegionMove1731707940151/b70e50bf67275a89944d13e9626080b2/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-15T21:59:00,621 INFO [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1114): Opened b70e50bf67275a89944d13e9626080b2; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=60670035, jitterRate=-0.09594602882862091}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-15T21:59:00,621 DEBUG [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1122): Running coprocessor post-open hooks for b70e50bf67275a89944d13e9626080b2 2024-11-15T21:59:00,623 DEBUG [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1006): Region open journal for b70e50bf67275a89944d13e9626080b2: Running coprocessor pre-open hook at 1731707940576Writing region info on filesystem at 1731707940576Initializing all the Stores at 1731707940576Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731707940576Cleaning up temporary data from old regions at 1731707940594 (+18 ms)Running coprocessor post-open hooks at 1731707940621 (+27 ms)Region opened successfully at 1731707940622 (+1 ms) 2024-11-15T21:59:00,624 INFO [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2236): Post open deploy tasks for testBlockEvictionOnRegionMove1731707940151,,1731707940181.b70e50bf67275a89944d13e9626080b2., pid=6, masterSystemTime=1731707940563 2024-11-15T21:59:00,628 DEBUG [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2266): Finished post open deploy task for testBlockEvictionOnRegionMove1731707940151,,1731707940181.b70e50bf67275a89944d13e9626080b2. 2024-11-15T21:59:00,629 INFO [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(153): Opened testBlockEvictionOnRegionMove1731707940151,,1731707940181.b70e50bf67275a89944d13e9626080b2. 2024-11-15T21:59:00,629 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=b70e50bf67275a89944d13e9626080b2, regionState=OPEN, openSeqNum=2, regionLocation=ae3311ea5ccb,35509,1731707937420 2024-11-15T21:59:00,633 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-4-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure b70e50bf67275a89944d13e9626080b2, server=ae3311ea5ccb,35509,1731707937420 because future has completed 2024-11-15T21:59:00,636 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=6, resume processing ppid=5 2024-11-15T21:59:00,637 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure b70e50bf67275a89944d13e9626080b2, server=ae3311ea5ccb,35509,1731707937420 in 224 msec 2024-11-15T21:59:00,639 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=5, resume processing ppid=4 2024-11-15T21:59:00,640 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1731707940151, region=b70e50bf67275a89944d13e9626080b2, ASSIGN in 389 msec 2024-11-15T21:59:00,641 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=testBlockEvictionOnRegionMove1731707940151 execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-15T21:59:00,641 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testBlockEvictionOnRegionMove1731707940151","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731707940641"}]},"ts":"1731707940641"} 2024-11-15T21:59:00,644 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=testBlockEvictionOnRegionMove1731707940151, state=ENABLED in hbase:meta 2024-11-15T21:59:00,646 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=testBlockEvictionOnRegionMove1731707940151 execute state=CREATE_TABLE_POST_OPERATION 2024-11-15T21:59:00,648 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=testBlockEvictionOnRegionMove1731707940151 in 457 msec 2024-11-15T21:59:00,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40769 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-15T21:59:00,838 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table testBlockEvictionOnRegionMove1731707940151 get assigned. Timeout = 60000ms 2024-11-15T21:59:00,838 INFO [RPCClient-NioEventLoopGroup-5-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:testBlockEvictionOnRegionMove1731707940151 completed 2024-11-15T21:59:00,841 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-15T21:59:00,847 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table testBlockEvictionOnRegionMove1731707940151 assigned to meta. Checking AM states. 2024-11-15T21:59:00,848 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-15T21:59:00,848 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table testBlockEvictionOnRegionMove1731707940151 assigned. 2024-11-15T21:59:00,858 DEBUG [RPCClient-NioEventLoopGroup-5-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testBlockEvictionOnRegionMove1731707940151', row='row1', locateType=CURRENT is [region=testBlockEvictionOnRegionMove1731707940151,,1731707940181.b70e50bf67275a89944d13e9626080b2., hostname=ae3311ea5ccb,35509,1731707937420, seqNum=2] 2024-11-15T21:59:00,859 DEBUG [RPCClient-NioEventLoopGroup-5-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-15T21:59:00,862 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:55650, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-15T21:59:00,872 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing b70e50bf67275a89944d13e9626080b2 1/1 column families, dataSize=78 B heapSize=480 B 2024-11-15T21:59:00,902 DEBUG [Time-limited test {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 894e8c613d5443b59a5fac5778a1b79f 2024-11-15T21:59:00,946 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3be1b200-2a9c-651b-3f6b-ac964062b7d6/f4b02001-5f5c-c384-44d5-c547c79a7f1b/data/default/testBlockEvictionOnRegionMove1731707940151/b70e50bf67275a89944d13e9626080b2/.tmp/family/894e8c613d5443b59a5fac5778a1b79f is 43, key is row1/family:qf1/1/Put/seqid=0 2024-11-15T21:59:00,954 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=78 B at sequenceid=6 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3be1b200-2a9c-651b-3f6b-ac964062b7d6/f4b02001-5f5c-c384-44d5-c547c79a7f1b/data/default/testBlockEvictionOnRegionMove1731707940151/b70e50bf67275a89944d13e9626080b2/.tmp/family/894e8c613d5443b59a5fac5778a1b79f 2024-11-15T21:59:01,009 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3be1b200-2a9c-651b-3f6b-ac964062b7d6/f4b02001-5f5c-c384-44d5-c547c79a7f1b/data/default/testBlockEvictionOnRegionMove1731707940151/b70e50bf67275a89944d13e9626080b2/.tmp/family/894e8c613d5443b59a5fac5778a1b79f as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3be1b200-2a9c-651b-3f6b-ac964062b7d6/f4b02001-5f5c-c384-44d5-c547c79a7f1b/data/default/testBlockEvictionOnRegionMove1731707940151/b70e50bf67275a89944d13e9626080b2/family/894e8c613d5443b59a5fac5778a1b79f 2024-11-15T21:59:01,011 DEBUG [Time-limited test {}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3be1b200-2a9c-651b-3f6b-ac964062b7d6/f4b02001-5f5c-c384-44d5-c547c79a7f1b/data/default/testBlockEvictionOnRegionMove1731707940151/b70e50bf67275a89944d13e9626080b2/family/894e8c613d5443b59a5fac5778a1b79f, delay=946 ms 2024-11-15T21:59:01,013 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3be1b200-2a9c-651b-3f6b-ac964062b7d6/f4b02001-5f5c-c384-44d5-c547c79a7f1b/data/default/testBlockEvictionOnRegionMove1731707940151/b70e50bf67275a89944d13e9626080b2/family/894e8c613d5443b59a5fac5778a1b79f, entries=2, sequenceid=6, filesize=4.9 K 2024-11-15T21:59:01,030 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~78 B/78, heapSize ~464 B/464, currentSize=0 B/0 for b70e50bf67275a89944d13e9626080b2 in 147ms, sequenceid=6, compaction requested=false 2024-11-15T21:59:01,032 DEBUG [Time-limited test {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'testBlockEvictionOnRegionMove1731707940151' 2024-11-15T21:59:01,036 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for b70e50bf67275a89944d13e9626080b2: 2024-11-15T21:59:01,960 DEBUG [hfile-prefetch-1731707941011 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 894e8c613d5443b59a5fac5778a1b79f_0. 2024-11-15T21:59:01,965 DEBUG [hfile-prefetch-1731707941011 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 894e8c613d5443b59a5fac5778a1b79f_117. 2024-11-15T21:59:01,967 DEBUG [hfile-prefetch-1731707941011 {}] bucket.BucketCache(2281): Notifying caching completed for file file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3be1b200-2a9c-651b-3f6b-ac964062b7d6/f4b02001-5f5c-c384-44d5-c547c79a7f1b/data/default/testBlockEvictionOnRegionMove1731707940151/b70e50bf67275a89944d13e9626080b2/family/894e8c613d5443b59a5fac5778a1b79f, with total blocks 2, and data blocks 1 2024-11-15T21:59:01,967 DEBUG [hfile-prefetch-1731707941011 {}] bucket.BucketCache(2285): iterating over 0 entries in the backing map 2024-11-15T21:59:01,967 DEBUG [hfile-prefetch-1731707941011 {}] bucket.BucketCache(2308): Prefetch executor completed for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3be1b200-2a9c-651b-3f6b-ac964062b7d6/f4b02001-5f5c-c384-44d5-c547c79a7f1b/data/default/testBlockEvictionOnRegionMove1731707940151/b70e50bf67275a89944d13e9626080b2/family/894e8c613d5443b59a5fac5778a1b79f, but only 0 data blocks were cached. Total data blocks for file: 1. Checking for blocks pending cache in cache writer queue. 2024-11-15T21:59:01,968 DEBUG [hfile-prefetch-1731707941011 {}] bucket.BucketCache(2318): There are still blocks pending caching for file file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3be1b200-2a9c-651b-3f6b-ac964062b7d6/f4b02001-5f5c-c384-44d5-c547c79a7f1b/data/default/testBlockEvictionOnRegionMove1731707940151/b70e50bf67275a89944d13e9626080b2/family/894e8c613d5443b59a5fac5778a1b79f. Will sleep 100ms and try the verification again. 2024-11-15T21:59:02,002 DEBUG [bucket-cache-persister {}] bucket.BucketCachePersister(47): Cache is inconsistent, persisting to disk 2024-11-15T21:59:02,002 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1487): Thread bucket-cache-persister started persisting bucket cache to file 2024-11-15T21:59:02,002 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1494): Persist in new chunked persistence format. 2024-11-15T21:59:02,002 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1673): persistToFile: before persisting backing map size: 0, fullycachedFiles size: 0, chunkSize: 10000 2024-11-15T21:59:02,012 DEBUG [bucket-cache-persister {}] bucket.BucketCachePersister(47): Cache is inconsistent, persisting to disk 2024-11-15T21:59:02,012 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1487): Thread bucket-cache-persister started persisting bucket cache to file 2024-11-15T21:59:02,012 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1494): Persist in new chunked persistence format. 2024-11-15T21:59:02,012 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1673): persistToFile: before persisting backing map size: 1, fullycachedFiles size: 0, chunkSize: 10000 2024-11-15T21:59:02,024 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1680): persistToFile: after persisting backing map size: 0, fullycachedFiles size: 0 2024-11-15T21:59:02,025 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1498): PersistToFile: after persisting backing map size: 0, fullycachedFiles size: 0, file name: bucket.persistence1731707942002 2024-11-15T21:59:02,025 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1509): Thread bucket-cache-persister finished persisting bucket cache to file, renaming 2024-11-15T21:59:02,028 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1680): persistToFile: after persisting backing map size: 1, fullycachedFiles size: 0 2024-11-15T21:59:02,029 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1498): PersistToFile: after persisting backing map size: 1, fullycachedFiles size: 0, file name: bucket.persistence1731707942012 2024-11-15T21:59:02,029 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1509): Thread bucket-cache-persister finished persisting bucket cache to file, renaming 2024-11-15T21:59:02,048 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40769 {}] master.HMaster(2410): Client=jenkins//172.17.0.3 move hri=b70e50bf67275a89944d13e9626080b2, source=ae3311ea5ccb,35509,1731707937420, destination=ae3311ea5ccb,32843,1731707937562, warming up region on ae3311ea5ccb,32843,1731707937562 2024-11-15T21:59:02,050 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40769 {}] master.HMaster(2414): Client=jenkins//172.17.0.3 move hri=b70e50bf67275a89944d13e9626080b2, source=ae3311ea5ccb,35509,1731707937420, destination=ae3311ea5ccb,32843,1731707937562, running balancer 2024-11-15T21:59:02,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40769 {}] procedure2.ProcedureExecutor(1139): Stored pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1731707940151, region=b70e50bf67275a89944d13e9626080b2, REOPEN/MOVE 2024-11-15T21:59:02,052 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1731707940151, region=b70e50bf67275a89944d13e9626080b2, REOPEN/MOVE 2024-11-15T21:59:02,052 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32843 {}] regionserver.RSRpcServices(2066): Warmup testBlockEvictionOnRegionMove1731707940151,,1731707940181.b70e50bf67275a89944d13e9626080b2. 2024-11-15T21:59:02,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32843 {}] regionserver.HRegion(7855): Warmup {ENCODED => b70e50bf67275a89944d13e9626080b2, NAME => 'testBlockEvictionOnRegionMove1731707940151,,1731707940181.b70e50bf67275a89944d13e9626080b2.', STARTKEY => '', ENDKEY => ''} 2024-11-15T21:59:02,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32843 {}] regionserver.HRegion(898): Instantiated testBlockEvictionOnRegionMove1731707940151,,1731707940181.b70e50bf67275a89944d13e9626080b2.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-15T21:59:02,054 INFO [StoreOpener-b70e50bf67275a89944d13e9626080b2-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family family of region b70e50bf67275a89944d13e9626080b2 2024-11-15T21:59:02,054 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=b70e50bf67275a89944d13e9626080b2, regionState=CLOSING, regionLocation=ae3311ea5ccb,35509,1731707937420 2024-11-15T21:59:02,055 INFO [StoreOpener-b70e50bf67275a89944d13e9626080b2-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region b70e50bf67275a89944d13e9626080b2 columnFamilyName family 2024-11-15T21:59:02,055 DEBUG [StoreOpener-b70e50bf67275a89944d13e9626080b2-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T21:59:02,058 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-4-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1731707940151, region=b70e50bf67275a89944d13e9626080b2, REOPEN/MOVE because future has completed 2024-11-15T21:59:02,059 DEBUG [PEWorker-3 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: true 2024-11-15T21:59:02,059 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=8, ppid=7, state=RUNNABLE, hasLock=false; CloseRegionProcedure b70e50bf67275a89944d13e9626080b2, server=ae3311ea5ccb,35509,1731707937420}] 2024-11-15T21:59:02,060 DEBUG [StoreOpener-b70e50bf67275a89944d13e9626080b2-1 {}] regionserver.StoreEngine(278): loaded file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3be1b200-2a9c-651b-3f6b-ac964062b7d6/f4b02001-5f5c-c384-44d5-c547c79a7f1b/data/default/testBlockEvictionOnRegionMove1731707940151/b70e50bf67275a89944d13e9626080b2/family/894e8c613d5443b59a5fac5778a1b79f 2024-11-15T21:59:02,060 INFO [StoreOpener-b70e50bf67275a89944d13e9626080b2-1 {}] regionserver.HStore(327): Store=b70e50bf67275a89944d13e9626080b2/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-15T21:59:02,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32843 {}] regionserver.HRegion(1722): Closing b70e50bf67275a89944d13e9626080b2, disabling compactions & flushes 2024-11-15T21:59:02,060 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32843 {}] regionserver.HRegion(1755): Closing region testBlockEvictionOnRegionMove1731707940151,,1731707940181.b70e50bf67275a89944d13e9626080b2. 2024-11-15T21:59:02,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32843 {}] regionserver.HRegion(1776): Time limited wait for close lock on testBlockEvictionOnRegionMove1731707940151,,1731707940181.b70e50bf67275a89944d13e9626080b2. 2024-11-15T21:59:02,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32843 {}] regionserver.HRegion(1843): Acquired close lock on testBlockEvictionOnRegionMove1731707940151,,1731707940181.b70e50bf67275a89944d13e9626080b2. after waiting 0 ms 2024-11-15T21:59:02,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32843 {}] regionserver.HRegion(1853): Updates disabled for region testBlockEvictionOnRegionMove1731707940151,,1731707940181.b70e50bf67275a89944d13e9626080b2. 2024-11-15T21:59:02,062 DEBUG [StoreFileCloser-b70e50bf67275a89944d13e9626080b2-family-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3be1b200-2a9c-651b-3f6b-ac964062b7d6/f4b02001-5f5c-c384-44d5-c547c79a7f1b/data/default/testBlockEvictionOnRegionMove1731707940151/b70e50bf67275a89944d13e9626080b2/family/894e8c613d5443b59a5fac5778a1b79f 2024-11-15T21:59:02,062 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32843 {}] regionserver.HRegion(1973): Closed testBlockEvictionOnRegionMove1731707940151,,1731707940181.b70e50bf67275a89944d13e9626080b2. 2024-11-15T21:59:02,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32843 {}] regionserver.HRegion(1676): Region close journal for b70e50bf67275a89944d13e9626080b2: Waiting for close lock at 1731707942060Disabling compacts and flushes for region at 1731707942060Disabling writes for close at 1731707942060Writing region close event to WAL at 1731707942062 (+2 ms)Closed at 1731707942062 2024-11-15T21:59:02,064 WARN [hfile-prefetch-1731707941011 {}] hfile.HFilePreadReader$1(158): Prefetch path=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3be1b200-2a9c-651b-3f6b-ac964062b7d6/f4b02001-5f5c-c384-44d5-c547c79a7f1b/data/default/testBlockEvictionOnRegionMove1731707940151/b70e50bf67275a89944d13e9626080b2/family/894e8c613d5443b59a5fac5778a1b79f, offset=158, end=158 java.lang.RuntimeException: java.lang.InterruptedException: sleep interrupted at org.apache.hadoop.hbase.io.hfile.bucket.BucketCache.notifyFileCachingCompleted(BucketCache.java:2337) ~[classes/:?] at org.apache.hadoop.hbase.io.hfile.CombinedBlockCache.notifyFileCachingCompleted(CombinedBlockCache.java:469) ~[classes/:?] at org.apache.hadoop.hbase.io.hfile.HFilePreadReader$1.run(HFilePreadReader.java:148) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.lambda$tracedRunnable$2(TraceUtil.java:155) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:539) ~[?:?] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:304) ~[?:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCache.notifyFileCachingCompleted(BucketCache.java:2320) ~[classes/:?] ... 9 more 2024-11-15T21:59:02,065 DEBUG [hfile-prefetch-1731707941011 {}] hfile.PrefetchExecutor(121): Prefetch completed for 894e8c613d5443b59a5fac5778a1b79f 2024-11-15T21:59:02,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40769 {}] procedure.ProcedureSyncWait(219): waitFor pid=7 2024-11-15T21:59:02,218 INFO [RS_CLOSE_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] handler.UnassignRegionHandler(122): Close b70e50bf67275a89944d13e9626080b2 2024-11-15T21:59:02,218 DEBUG [RS_CLOSE_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: true 2024-11-15T21:59:02,219 DEBUG [RS_CLOSE_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1722): Closing b70e50bf67275a89944d13e9626080b2, disabling compactions & flushes 2024-11-15T21:59:02,219 INFO [RS_CLOSE_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1755): Closing region testBlockEvictionOnRegionMove1731707940151,,1731707940181.b70e50bf67275a89944d13e9626080b2. 2024-11-15T21:59:02,219 DEBUG [RS_CLOSE_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1776): Time limited wait for close lock on testBlockEvictionOnRegionMove1731707940151,,1731707940181.b70e50bf67275a89944d13e9626080b2. 2024-11-15T21:59:02,219 DEBUG [RS_CLOSE_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1843): Acquired close lock on testBlockEvictionOnRegionMove1731707940151,,1731707940181.b70e50bf67275a89944d13e9626080b2. after waiting 0 ms 2024-11-15T21:59:02,219 DEBUG [RS_CLOSE_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1853): Updates disabled for region testBlockEvictionOnRegionMove1731707940151,,1731707940181.b70e50bf67275a89944d13e9626080b2. 2024-11-15T21:59:02,221 DEBUG [StoreFileCloser-b70e50bf67275a89944d13e9626080b2-family-1 {}] bucket.BucketCache(1863): found 1 blocks for file 894e8c613d5443b59a5fac5778a1b79f, starting offset: 0, end offset: 9223372036854775807 2024-11-15T21:59:02,222 DEBUG [StoreFileCloser-b70e50bf67275a89944d13e9626080b2-family-1 {}] bucket.BucketCache(856): removed key 894e8c613d5443b59a5fac5778a1b79f_0 from back map with offset lock 2094080 in the evict process 2024-11-15T21:59:02,225 DEBUG [bucket-cache-persister {}] bucket.BucketCachePersister(47): Cache is inconsistent, persisting to disk 2024-11-15T21:59:02,225 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1487): Thread bucket-cache-persister started persisting bucket cache to file 2024-11-15T21:59:02,225 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1494): Persist in new chunked persistence format. 2024-11-15T21:59:02,225 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1673): persistToFile: before persisting backing map size: 0, fullycachedFiles size: 0, chunkSize: 10000 2024-11-15T21:59:02,228 DEBUG [RS_CLOSE_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3be1b200-2a9c-651b-3f6b-ac964062b7d6/f4b02001-5f5c-c384-44d5-c547c79a7f1b/data/default/testBlockEvictionOnRegionMove1731707940151/b70e50bf67275a89944d13e9626080b2/recovered.edits/9.seqid, newMaxSeqId=9, maxSeqId=1 2024-11-15T21:59:02,229 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1680): persistToFile: after persisting backing map size: 0, fullycachedFiles size: 0 2024-11-15T21:59:02,229 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1498): PersistToFile: after persisting backing map size: 0, fullycachedFiles size: 0, file name: bucket.persistence1731707942225 2024-11-15T21:59:02,229 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1509): Thread bucket-cache-persister finished persisting bucket cache to file, renaming 2024-11-15T21:59:02,230 INFO [RS_CLOSE_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1973): Closed testBlockEvictionOnRegionMove1731707940151,,1731707940181.b70e50bf67275a89944d13e9626080b2. 2024-11-15T21:59:02,230 DEBUG [RS_CLOSE_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1676): Region close journal for b70e50bf67275a89944d13e9626080b2: Waiting for close lock at 1731707942219Running coprocessor pre-close hooks at 1731707942219Disabling compacts and flushes for region at 1731707942219Disabling writes for close at 1731707942219Writing region close event to WAL at 1731707942223 (+4 ms)Running coprocessor post-close hooks at 1731707942228 (+5 ms)Closed at 1731707942230 (+2 ms) 2024-11-15T21:59:02,230 INFO [RS_CLOSE_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegionServer(3302): Adding b70e50bf67275a89944d13e9626080b2 move to ae3311ea5ccb,32843,1731707937562 record at close sequenceid=6 2024-11-15T21:59:02,234 INFO [RS_CLOSE_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] handler.UnassignRegionHandler(157): Closed b70e50bf67275a89944d13e9626080b2 2024-11-15T21:59:02,234 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=b70e50bf67275a89944d13e9626080b2, regionState=CLOSED 2024-11-15T21:59:02,236 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-4-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=8, ppid=7, state=RUNNABLE, hasLock=false; CloseRegionProcedure b70e50bf67275a89944d13e9626080b2, server=ae3311ea5ccb,35509,1731707937420 because future has completed 2024-11-15T21:59:02,240 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=8, resume processing ppid=7 2024-11-15T21:59:02,240 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=8, ppid=7, state=SUCCESS, hasLock=false; CloseRegionProcedure b70e50bf67275a89944d13e9626080b2, server=ae3311ea5ccb,35509,1731707937420 in 179 msec 2024-11-15T21:59:02,241 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1731707940151, region=b70e50bf67275a89944d13e9626080b2, REOPEN/MOVE; state=CLOSED, location=ae3311ea5ccb,32843,1731707937562; forceNewPlan=false, retain=false 2024-11-15T21:59:02,392 INFO [ae3311ea5ccb:40769 {}] balancer.BaseLoadBalancer(388): Reassigned 1 regions. 1 retained the pre-restart assignment. 2024-11-15T21:59:02,392 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=b70e50bf67275a89944d13e9626080b2, regionState=OPENING, regionLocation=ae3311ea5ccb,32843,1731707937562 2024-11-15T21:59:02,396 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-4-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1731707940151, region=b70e50bf67275a89944d13e9626080b2, REOPEN/MOVE because future has completed 2024-11-15T21:59:02,397 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=9, ppid=7, state=RUNNABLE, hasLock=false; OpenRegionProcedure b70e50bf67275a89944d13e9626080b2, server=ae3311ea5ccb,32843,1731707937562}] 2024-11-15T21:59:02,553 INFO [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] handler.AssignRegionHandler(132): Open testBlockEvictionOnRegionMove1731707940151,,1731707940181.b70e50bf67275a89944d13e9626080b2. 2024-11-15T21:59:02,553 DEBUG [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(7752): Opening region: {ENCODED => b70e50bf67275a89944d13e9626080b2, NAME => 'testBlockEvictionOnRegionMove1731707940151,,1731707940181.b70e50bf67275a89944d13e9626080b2.', STARTKEY => '', ENDKEY => ''} 2024-11-15T21:59:02,553 DEBUG [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testBlockEvictionOnRegionMove1731707940151 b70e50bf67275a89944d13e9626080b2 2024-11-15T21:59:02,553 DEBUG [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(898): Instantiated testBlockEvictionOnRegionMove1731707940151,,1731707940181.b70e50bf67275a89944d13e9626080b2.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-15T21:59:02,554 DEBUG [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(7794): checking encryption for b70e50bf67275a89944d13e9626080b2 2024-11-15T21:59:02,554 DEBUG [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(7797): checking classloading for b70e50bf67275a89944d13e9626080b2 2024-11-15T21:59:02,555 INFO [StoreOpener-b70e50bf67275a89944d13e9626080b2-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family family of region b70e50bf67275a89944d13e9626080b2 2024-11-15T21:59:02,556 INFO [StoreOpener-b70e50bf67275a89944d13e9626080b2-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region b70e50bf67275a89944d13e9626080b2 columnFamilyName family 2024-11-15T21:59:02,556 DEBUG [StoreOpener-b70e50bf67275a89944d13e9626080b2-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T21:59:02,558 DEBUG [StoreFileOpener-b70e50bf67275a89944d13e9626080b2-family-1 {}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3be1b200-2a9c-651b-3f6b-ac964062b7d6/f4b02001-5f5c-c384-44d5-c547c79a7f1b/data/default/testBlockEvictionOnRegionMove1731707940151/b70e50bf67275a89944d13e9626080b2/family/894e8c613d5443b59a5fac5778a1b79f, delay=959 ms 2024-11-15T21:59:02,560 DEBUG [StoreOpener-b70e50bf67275a89944d13e9626080b2-1 {}] regionserver.StoreEngine(278): loaded file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3be1b200-2a9c-651b-3f6b-ac964062b7d6/f4b02001-5f5c-c384-44d5-c547c79a7f1b/data/default/testBlockEvictionOnRegionMove1731707940151/b70e50bf67275a89944d13e9626080b2/family/894e8c613d5443b59a5fac5778a1b79f 2024-11-15T21:59:02,560 INFO [StoreOpener-b70e50bf67275a89944d13e9626080b2-1 {}] regionserver.HStore(327): Store=b70e50bf67275a89944d13e9626080b2/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-15T21:59:02,560 DEBUG [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1038): replaying wal for b70e50bf67275a89944d13e9626080b2 2024-11-15T21:59:02,561 DEBUG [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3be1b200-2a9c-651b-3f6b-ac964062b7d6/f4b02001-5f5c-c384-44d5-c547c79a7f1b/data/default/testBlockEvictionOnRegionMove1731707940151/b70e50bf67275a89944d13e9626080b2 2024-11-15T21:59:02,561 DEBUG [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3be1b200-2a9c-651b-3f6b-ac964062b7d6/f4b02001-5f5c-c384-44d5-c547c79a7f1b/data/default/testBlockEvictionOnRegionMove1731707940151/b70e50bf67275a89944d13e9626080b2 2024-11-15T21:59:02,561 DEBUG [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1048): stopping wal replay for b70e50bf67275a89944d13e9626080b2 2024-11-15T21:59:02,561 DEBUG [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1060): Cleaning up temporary data for b70e50bf67275a89944d13e9626080b2 2024-11-15T21:59:02,566 DEBUG [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1093): writing seq id for b70e50bf67275a89944d13e9626080b2 2024-11-15T21:59:02,566 INFO [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1114): Opened b70e50bf67275a89944d13e9626080b2; next sequenceid=10; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=65195083, jitterRate=-0.02851755917072296}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-15T21:59:02,566 DEBUG [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1122): Running coprocessor post-open hooks for b70e50bf67275a89944d13e9626080b2 2024-11-15T21:59:02,567 DEBUG [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1006): Region open journal for b70e50bf67275a89944d13e9626080b2: Running coprocessor pre-open hook at 1731707942554Writing region info on filesystem at 1731707942554Initializing all the Stores at 1731707942555 (+1 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731707942555Cleaning up temporary data from old regions at 1731707942561 (+6 ms)Running coprocessor post-open hooks at 1731707942566 (+5 ms)Region opened successfully at 1731707942567 (+1 ms) 2024-11-15T21:59:02,568 INFO [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegionServer(2236): Post open deploy tasks for testBlockEvictionOnRegionMove1731707940151,,1731707940181.b70e50bf67275a89944d13e9626080b2., pid=9, masterSystemTime=1731707942549 2024-11-15T21:59:02,572 DEBUG [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegionServer(2266): Finished post open deploy task for testBlockEvictionOnRegionMove1731707940151,,1731707940181.b70e50bf67275a89944d13e9626080b2. 2024-11-15T21:59:02,572 INFO [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] handler.AssignRegionHandler(153): Opened testBlockEvictionOnRegionMove1731707940151,,1731707940181.b70e50bf67275a89944d13e9626080b2. 2024-11-15T21:59:02,572 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=b70e50bf67275a89944d13e9626080b2, regionState=OPEN, openSeqNum=10, regionLocation=ae3311ea5ccb,32843,1731707937562 2024-11-15T21:59:02,575 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-4-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=9, ppid=7, state=RUNNABLE, hasLock=false; OpenRegionProcedure b70e50bf67275a89944d13e9626080b2, server=ae3311ea5ccb,32843,1731707937562 because future has completed 2024-11-15T21:59:02,578 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=9, resume processing ppid=7 2024-11-15T21:59:02,578 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=9, ppid=7, state=SUCCESS, hasLock=false; OpenRegionProcedure b70e50bf67275a89944d13e9626080b2, server=ae3311ea5ccb,32843,1731707937562 in 179 msec 2024-11-15T21:59:02,579 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=7, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1731707940151, region=b70e50bf67275a89944d13e9626080b2, REOPEN/MOVE in 527 msec 2024-11-15T21:59:02,605 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster 2024-11-15T21:59:02,605 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-15T21:59:02,606 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79) at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611) at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.io.hfile.TestBlockEvictionOnRegionMovement.tearDown(TestBlockEvictionOnRegionMovement.java:174) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-15T21:59:02,611 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T21:59:02,612 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T21:59:02,612 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-15T21:59:02,612 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2024-11-15T21:59:02,612 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=587878705, stopped=false 2024-11-15T21:59:02,613 INFO [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=ae3311ea5ccb,40769,1731707936561 2024-11-15T21:59:02,616 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40769-0x100f86912970000, quorum=127.0.0.1:62797, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-15T21:59:02,616 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:32843-0x100f86912970002, quorum=127.0.0.1:62797, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-15T21:59:02,616 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35509-0x100f86912970001, quorum=127.0.0.1:62797, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-15T21:59:02,616 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35509-0x100f86912970001, quorum=127.0.0.1:62797, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-15T21:59:02,616 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40769-0x100f86912970000, quorum=127.0.0.1:62797, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-15T21:59:02,616 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:32843-0x100f86912970002, quorum=127.0.0.1:62797, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-15T21:59:02,616 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-15T21:59:02,617 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:32843-0x100f86912970002, quorum=127.0.0.1:62797, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-15T21:59:02,617 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:35509-0x100f86912970001, quorum=127.0.0.1:62797, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-15T21:59:02,618 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-15T21:59:02,618 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277) at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265) at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416) at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.io.hfile.TestBlockEvictionOnRegionMovement.tearDown(TestBlockEvictionOnRegionMovement.java:174) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-15T21:59:02,618 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T21:59:02,618 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:40769-0x100f86912970000, quorum=127.0.0.1:62797, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-15T21:59:02,618 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server 'ae3311ea5ccb,35509,1731707937420' ***** 2024-11-15T21:59:02,618 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-11-15T21:59:02,618 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server 'ae3311ea5ccb,32843,1731707937562' ***** 2024-11-15T21:59:02,619 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-11-15T21:59:02,619 INFO [RS:0;ae3311ea5ccb:35509 {}] bucket.BucketCache(1738): Disabling cache 2024-11-15T21:59:02,619 INFO [RS:1;ae3311ea5ccb:32843 {}] bucket.BucketCache(1738): Disabling cache 2024-11-15T21:59:02,619 INFO [RS:0;ae3311ea5ccb:35509 {}] bucket.BucketCache(1762): Shutdown bucket cache: IO persistent=true; path to write=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3be1b200-2a9c-651b-3f6b-ac964062b7d6/bucket.persistence 2024-11-15T21:59:02,619 INFO [Time-limited test-BucketCacheWriter-1 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-1 exiting, cacheEnabled=false 2024-11-15T21:59:02,619 INFO [Time-limited test-BucketCacheWriter-0 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-0 exiting, cacheEnabled=false 2024-11-15T21:59:02,619 INFO [Time-limited test-BucketCacheWriter-0 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-0 exiting, cacheEnabled=false 2024-11-15T21:59:02,619 INFO [Time-limited test-BucketCacheWriter-1 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-1 exiting, cacheEnabled=false 2024-11-15T21:59:02,620 INFO [Time-limited test-BucketCacheWriter-2 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-2 exiting, cacheEnabled=false 2024-11-15T21:59:02,620 INFO [RS:0;ae3311ea5ccb:35509 {}] bucket.BucketCache(1768): Shutting down cache persister thread. 2024-11-15T21:59:02,620 WARN [bucket-cache-persister {}] bucket.BucketCachePersister(64): Interrupting BucketCachePersister thread. java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCachePersister.run(BucketCachePersister.java:45) ~[classes/:?] 2024-11-15T21:59:02,620 INFO [RS:1;ae3311ea5ccb:32843 {}] bucket.BucketCache(1762): Shutdown bucket cache: IO persistent=true; path to write=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3be1b200-2a9c-651b-3f6b-ac964062b7d6/bucket.persistence 2024-11-15T21:59:02,620 INFO [Time-limited test-BucketCacheWriter-2 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-2 exiting, cacheEnabled=false 2024-11-15T21:59:02,620 INFO [RS:1;ae3311ea5ccb:32843 {}] bucket.BucketCache(1768): Shutting down cache persister thread. 2024-11-15T21:59:02,620 WARN [bucket-cache-persister {}] bucket.BucketCachePersister(64): Interrupting BucketCachePersister thread. java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCachePersister.run(BucketCachePersister.java:45) ~[classes/:?] 2024-11-15T21:59:02,630 DEBUG [RS:0;ae3311ea5ccb:35509 {}] bucket.BucketCache(1487): Thread RS:0;ae3311ea5ccb:35509 started persisting bucket cache to file 2024-11-15T21:59:02,630 DEBUG [RS:0;ae3311ea5ccb:35509 {}] bucket.BucketCache(1494): Persist in new chunked persistence format. 2024-11-15T21:59:02,630 DEBUG [RS:0;ae3311ea5ccb:35509 {}] bucket.BucketCache(1673): persistToFile: before persisting backing map size: 0, fullycachedFiles size: 0, chunkSize: 10000 2024-11-15T21:59:02,630 DEBUG [RS:1;ae3311ea5ccb:32843 {}] bucket.BucketCache(1487): Thread RS:1;ae3311ea5ccb:32843 started persisting bucket cache to file 2024-11-15T21:59:02,631 DEBUG [RS:1;ae3311ea5ccb:32843 {}] bucket.BucketCache(1494): Persist in new chunked persistence format. 2024-11-15T21:59:02,631 DEBUG [RS:1;ae3311ea5ccb:32843 {}] bucket.BucketCache(1673): persistToFile: before persisting backing map size: 0, fullycachedFiles size: 0, chunkSize: 10000 2024-11-15T21:59:02,635 DEBUG [RS:0;ae3311ea5ccb:35509 {}] bucket.BucketCache(1680): persistToFile: after persisting backing map size: 0, fullycachedFiles size: 0 2024-11-15T21:59:02,635 DEBUG [RS:0;ae3311ea5ccb:35509 {}] bucket.BucketCache(1498): PersistToFile: after persisting backing map size: 0, fullycachedFiles size: 0, file name: bucket.persistence1731707942630 2024-11-15T21:59:02,635 DEBUG [RS:0;ae3311ea5ccb:35509 {}] bucket.BucketCache(1509): Thread RS:0;ae3311ea5ccb:35509 finished persisting bucket cache to file, renaming 2024-11-15T21:59:02,635 INFO [RS:0;ae3311ea5ccb:35509 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-15T21:59:02,635 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-15T21:59:02,635 INFO [RS:0;ae3311ea5ccb:35509 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-15T21:59:02,636 INFO [RS:0;ae3311ea5ccb:35509 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-15T21:59:02,636 INFO [RS:0;ae3311ea5ccb:35509 {}] regionserver.HRegionServer(959): stopping server ae3311ea5ccb,35509,1731707937420 2024-11-15T21:59:02,636 INFO [RS:0;ae3311ea5ccb:35509 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-15T21:59:02,636 INFO [RS:0;ae3311ea5ccb:35509 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;ae3311ea5ccb:35509. 2024-11-15T21:59:02,636 DEBUG [RS:0;ae3311ea5ccb:35509 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-15T21:59:02,636 DEBUG [RS:0;ae3311ea5ccb:35509 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T21:59:02,636 INFO [RS:0;ae3311ea5ccb:35509 {}] regionserver.HRegionServer(976): stopping server ae3311ea5ccb,35509,1731707937420; all regions closed. 2024-11-15T21:59:02,637 WARN [Thread-96 {}] util.Shell$1(1036): Error reading the error stream java.io.IOException: Stream closed at java.io.BufferedInputStream.getBufIfOpen(BufferedInputStream.java:168) ~[?:?] at java.io.BufferedInputStream.read(BufferedInputStream.java:334) ~[?:?] at sun.nio.cs.StreamDecoder.readBytes(StreamDecoder.java:281) ~[?:?] at sun.nio.cs.StreamDecoder.implRead(StreamDecoder.java:324) ~[?:?] at sun.nio.cs.StreamDecoder.read(StreamDecoder.java:189) ~[?:?] at java.io.InputStreamReader.read(InputStreamReader.java:177) ~[?:?] at java.io.BufferedReader.fill(BufferedReader.java:162) ~[?:?] at java.io.BufferedReader.readLine(BufferedReader.java:329) ~[?:?] at java.io.BufferedReader.readLine(BufferedReader.java:396) ~[?:?] at org.apache.hadoop.util.Shell$1.run(Shell.java:1025) ~[hadoop-common-3.4.1.jar:?] 2024-11-15T21:59:02,637 ERROR [RS:1;ae3311ea5ccb:32843 {}] bucket.PersistentIOEngine(78): Calculating checksum failed, because of java.io.IOException: Stream closed at java.io.BufferedInputStream.getBufIfOpen(BufferedInputStream.java:168) ~[?:?] at java.io.BufferedInputStream.read(BufferedInputStream.java:334) ~[?:?] at sun.nio.cs.StreamDecoder.readBytes(StreamDecoder.java:281) ~[?:?] at sun.nio.cs.StreamDecoder.implRead(StreamDecoder.java:324) ~[?:?] at sun.nio.cs.StreamDecoder.read(StreamDecoder.java:189) ~[?:?] at java.io.InputStreamReader.read(InputStreamReader.java:177) ~[?:?] at java.io.BufferedReader.fill(BufferedReader.java:162) ~[?:?] at java.io.BufferedReader.read1(BufferedReader.java:213) ~[?:?] at java.io.BufferedReader.read(BufferedReader.java:287) ~[?:?] at org.apache.hadoop.util.Shell$ShellCommandExecutor.parseExecResult(Shell.java:1295) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.util.Shell.runCommand(Shell.java:1054) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.util.Shell.run(Shell.java:959) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.util.Shell$ShellCommandExecutor.execute(Shell.java:1282) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.io.hfile.bucket.PersistentIOEngine.getFileSize(PersistentIOEngine.java:94) ~[classes/:?] at org.apache.hadoop.hbase.io.hfile.bucket.PersistentIOEngine.calculateChecksum(PersistentIOEngine.java:71) ~[classes/:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketProtoUtils.toPB(BucketProtoUtils.java:61) ~[classes/:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketProtoUtils.serializeAsPB(BucketProtoUtils.java:75) ~[classes/:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCache.persistChunkedBackingMap(BucketCache.java:1678) ~[classes/:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCache.persistToFile(BucketCache.java:1496) ~[classes/:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCache.shutdown(BucketCache.java:1774) ~[classes/:?] at org.apache.hadoop.hbase.io.hfile.CombinedBlockCache.shutdown(CombinedBlockCache.java:169) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:927) ~[classes/:?] at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) ~[test-classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) ~[test-classes/:4.0.0-alpha-1-SNAPSHOT] at java.security.AccessController.doPrivileged(AccessController.java:399) ~[?:?] at javax.security.auth.Subject.doAs(Subject.java:376) ~[?:?] at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) ~[test-classes/:4.0.0-alpha-1-SNAPSHOT] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T21:59:02,638 DEBUG [RS:1;ae3311ea5ccb:32843 {}] bucket.BucketCache(1680): persistToFile: after persisting backing map size: 0, fullycachedFiles size: 0 2024-11-15T21:59:02,638 DEBUG [RS:1;ae3311ea5ccb:32843 {}] bucket.BucketCache(1498): PersistToFile: after persisting backing map size: 0, fullycachedFiles size: 0, file name: bucket.persistence1731707942630 2024-11-15T21:59:02,638 DEBUG [RS:1;ae3311ea5ccb:32843 {}] bucket.BucketCache(1509): Thread RS:1;ae3311ea5ccb:32843 finished persisting bucket cache to file, renaming 2024-11-15T21:59:02,638 WARN [RS:1;ae3311ea5ccb:32843 {}] bucket.BucketCache(1512): Failed to commit cache persistent file. We might lose cached blocks if RS crashes/restarts before we successfully checkpoint again. 2024-11-15T21:59:02,638 INFO [RS:1;ae3311ea5ccb:32843 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-15T21:59:02,638 INFO [RS:1;ae3311ea5ccb:32843 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-15T21:59:02,638 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-15T21:59:02,639 INFO [RS:1;ae3311ea5ccb:32843 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-15T21:59:02,639 INFO [RS:1;ae3311ea5ccb:32843 {}] regionserver.HRegionServer(3091): Received CLOSE for b70e50bf67275a89944d13e9626080b2 2024-11-15T21:59:02,639 INFO [RS:1;ae3311ea5ccb:32843 {}] regionserver.HRegionServer(959): stopping server ae3311ea5ccb,32843,1731707937562 2024-11-15T21:59:02,639 INFO [RS:1;ae3311ea5ccb:32843 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-15T21:59:02,640 INFO [RS:1;ae3311ea5ccb:32843 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:1;ae3311ea5ccb:32843. 2024-11-15T21:59:02,640 DEBUG [RS:1;ae3311ea5ccb:32843 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-15T21:59:02,640 DEBUG [RS:1;ae3311ea5ccb:32843 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T21:59:02,640 INFO [RS_CLOSE_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1662): Closing region testBlockEvictionOnRegionMove1731707940151,,1731707940181.b70e50bf67275a89944d13e9626080b2. during a graceful stop, and cache persistence is on, so setting evict on close to false. 2024-11-15T21:59:02,640 INFO [RS:1;ae3311ea5ccb:32843 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-15T21:59:02,640 INFO [RS:1;ae3311ea5ccb:32843 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-15T21:59:02,640 INFO [RS:1;ae3311ea5ccb:32843 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-15T21:59:02,640 INFO [RS:1;ae3311ea5ccb:32843 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-11-15T21:59:02,640 DEBUG [RS_CLOSE_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing b70e50bf67275a89944d13e9626080b2, disabling compactions & flushes 2024-11-15T21:59:02,640 INFO [RS_CLOSE_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testBlockEvictionOnRegionMove1731707940151,,1731707940181.b70e50bf67275a89944d13e9626080b2. 2024-11-15T21:59:02,640 DEBUG [RS_CLOSE_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testBlockEvictionOnRegionMove1731707940151,,1731707940181.b70e50bf67275a89944d13e9626080b2. 2024-11-15T21:59:02,641 DEBUG [RS_CLOSE_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testBlockEvictionOnRegionMove1731707940151,,1731707940181.b70e50bf67275a89944d13e9626080b2. after waiting 0 ms 2024-11-15T21:59:02,641 DEBUG [RS_CLOSE_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testBlockEvictionOnRegionMove1731707940151,,1731707940181.b70e50bf67275a89944d13e9626080b2. 2024-11-15T21:59:02,641 INFO [RS:1;ae3311ea5ccb:32843 {}] regionserver.HRegionServer(1321): Waiting on 2 regions to close 2024-11-15T21:59:02,641 DEBUG [RS:1;ae3311ea5ccb:32843 {}] regionserver.HRegionServer(1325): Online Regions={1588230740=hbase:meta,,1.1588230740, b70e50bf67275a89944d13e9626080b2=testBlockEvictionOnRegionMove1731707940151,,1731707940181.b70e50bf67275a89944d13e9626080b2.} 2024-11-15T21:59:02,641 INFO [RS_CLOSE_META-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1662): Closing region hbase:meta,,1.1588230740 during a graceful stop, and cache persistence is on, so setting evict on close to false. 2024-11-15T21:59:02,641 DEBUG [RS_CLOSE_META-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-15T21:59:02,641 INFO [RS_CLOSE_META-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-15T21:59:02,641 DEBUG [RS:1;ae3311ea5ccb:32843 {}] regionserver.HRegionServer(1351): Waiting on 1588230740, b70e50bf67275a89944d13e9626080b2 2024-11-15T21:59:02,641 DEBUG [RS_CLOSE_META-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-15T21:59:02,641 DEBUG [RS_CLOSE_META-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-15T21:59:02,641 DEBUG [RS_CLOSE_META-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-15T21:59:02,642 DEBUG [RS:0;ae3311ea5ccb:35509 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3be1b200-2a9c-651b-3f6b-ac964062b7d6/f4b02001-5f5c-c384-44d5-c547c79a7f1b/oldWALs 2024-11-15T21:59:02,642 INFO [RS_CLOSE_META-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=3.70 KB heapSize=6.72 KB 2024-11-15T21:59:02,642 INFO [RS:0;ae3311ea5ccb:35509 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL ae3311ea5ccb%2C35509%2C1731707937420:(num 1731707939390) 2024-11-15T21:59:02,642 DEBUG [StoreFileCloser-b70e50bf67275a89944d13e9626080b2-family-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3be1b200-2a9c-651b-3f6b-ac964062b7d6/f4b02001-5f5c-c384-44d5-c547c79a7f1b/data/default/testBlockEvictionOnRegionMove1731707940151/b70e50bf67275a89944d13e9626080b2/family/894e8c613d5443b59a5fac5778a1b79f 2024-11-15T21:59:02,642 DEBUG [RS:0;ae3311ea5ccb:35509 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T21:59:02,642 INFO [RS:0;ae3311ea5ccb:35509 {}] regionserver.LeaseManager(133): Closed leases 2024-11-15T21:59:02,642 INFO [RS:0;ae3311ea5ccb:35509 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-15T21:59:02,643 INFO [RS:0;ae3311ea5ccb:35509 {}] hbase.ChoreService(370): Chore service for: regionserver/ae3311ea5ccb:0 had [ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS] on shutdown 2024-11-15T21:59:02,643 INFO [regionserver/ae3311ea5ccb:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-15T21:59:02,644 INFO [RS:0;ae3311ea5ccb:35509 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-15T21:59:02,644 INFO [RS:0;ae3311ea5ccb:35509 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-15T21:59:02,644 INFO [RS:0;ae3311ea5ccb:35509 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-15T21:59:02,644 INFO [RS:0;ae3311ea5ccb:35509 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-15T21:59:02,645 INFO [RS:0;ae3311ea5ccb:35509 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.3:35509 2024-11-15T21:59:02,650 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35509-0x100f86912970001, quorum=127.0.0.1:62797, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/ae3311ea5ccb,35509,1731707937420 2024-11-15T21:59:02,650 INFO [RS:0;ae3311ea5ccb:35509 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-15T21:59:02,650 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40769-0x100f86912970000, quorum=127.0.0.1:62797, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-15T21:59:02,651 DEBUG [RS_CLOSE_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3be1b200-2a9c-651b-3f6b-ac964062b7d6/f4b02001-5f5c-c384-44d5-c547c79a7f1b/data/default/testBlockEvictionOnRegionMove1731707940151/b70e50bf67275a89944d13e9626080b2/recovered.edits/12.seqid, newMaxSeqId=12, maxSeqId=9 2024-11-15T21:59:02,651 INFO [RS_CLOSE_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testBlockEvictionOnRegionMove1731707940151,,1731707940181.b70e50bf67275a89944d13e9626080b2. 2024-11-15T21:59:02,651 DEBUG [RS_CLOSE_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for b70e50bf67275a89944d13e9626080b2: Waiting for close lock at 1731707942640Running coprocessor pre-close hooks at 1731707942640Disabling compacts and flushes for region at 1731707942640Disabling writes for close at 1731707942641 (+1 ms)Writing region close event to WAL at 1731707942643 (+2 ms)Running coprocessor post-close hooks at 1731707942651 (+8 ms)Closed at 1731707942651 2024-11-15T21:59:02,652 DEBUG [RS_CLOSE_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testBlockEvictionOnRegionMove1731707940151,,1731707940181.b70e50bf67275a89944d13e9626080b2. 2024-11-15T21:59:02,653 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [ae3311ea5ccb,35509,1731707937420] 2024-11-15T21:59:02,655 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/ae3311ea5ccb,35509,1731707937420 already deleted, retry=false 2024-11-15T21:59:02,655 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; ae3311ea5ccb,35509,1731707937420 expired; onlineServers=1 2024-11-15T21:59:02,656 DEBUG [RS_CLOSE_META-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 8b7b658c9d864b259c85da4f26b21bb9 2024-11-15T21:59:02,674 DEBUG [RS_CLOSE_META-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3be1b200-2a9c-651b-3f6b-ac964062b7d6/f4b02001-5f5c-c384-44d5-c547c79a7f1b/data/hbase/meta/1588230740/.tmp/info/8b7b658c9d864b259c85da4f26b21bb9 is 205, key is testBlockEvictionOnRegionMove1731707940151,,1731707940181.b70e50bf67275a89944d13e9626080b2./info:regioninfo/1731707942572/Put/seqid=0 2024-11-15T21:59:02,674 INFO [RS_CLOSE_META-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=3.48 KB at sequenceid=15 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3be1b200-2a9c-651b-3f6b-ac964062b7d6/f4b02001-5f5c-c384-44d5-c547c79a7f1b/data/hbase/meta/1588230740/.tmp/info/8b7b658c9d864b259c85da4f26b21bb9 2024-11-15T21:59:02,685 DEBUG [RS_CLOSE_META-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 547a163437dd4a3886ce4290badb63ae 2024-11-15T21:59:02,698 DEBUG [RS_CLOSE_META-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3be1b200-2a9c-651b-3f6b-ac964062b7d6/f4b02001-5f5c-c384-44d5-c547c79a7f1b/data/hbase/meta/1588230740/.tmp/ns/547a163437dd4a3886ce4290badb63ae is 43, key is default/ns:d/1731707939934/Put/seqid=0 2024-11-15T21:59:02,699 INFO [RS_CLOSE_META-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=74 B at sequenceid=15 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3be1b200-2a9c-651b-3f6b-ac964062b7d6/f4b02001-5f5c-c384-44d5-c547c79a7f1b/data/hbase/meta/1588230740/.tmp/ns/547a163437dd4a3886ce4290badb63ae 2024-11-15T21:59:02,709 DEBUG [RS_CLOSE_META-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFile$WriterFactory(308): Unable to set drop behind on a7692679ab064b2384c9ffda13eb9965 2024-11-15T21:59:02,715 INFO [regionserver/ae3311ea5ccb:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-15T21:59:02,718 INFO [regionserver/ae3311ea5ccb:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-15T21:59:02,723 DEBUG [RS_CLOSE_META-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3be1b200-2a9c-651b-3f6b-ac964062b7d6/f4b02001-5f5c-c384-44d5-c547c79a7f1b/data/hbase/meta/1588230740/.tmp/table/a7692679ab064b2384c9ffda13eb9965 is 78, key is testBlockEvictionOnRegionMove1731707940151/table:state/1731707940641/Put/seqid=0 2024-11-15T21:59:02,724 INFO [RS_CLOSE_META-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=148 B at sequenceid=15 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3be1b200-2a9c-651b-3f6b-ac964062b7d6/f4b02001-5f5c-c384-44d5-c547c79a7f1b/data/hbase/meta/1588230740/.tmp/table/a7692679ab064b2384c9ffda13eb9965 2024-11-15T21:59:02,726 DEBUG [RS_CLOSE_META-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3be1b200-2a9c-651b-3f6b-ac964062b7d6/f4b02001-5f5c-c384-44d5-c547c79a7f1b/data/hbase/meta/1588230740/.tmp/info/8b7b658c9d864b259c85da4f26b21bb9 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3be1b200-2a9c-651b-3f6b-ac964062b7d6/f4b02001-5f5c-c384-44d5-c547c79a7f1b/data/hbase/meta/1588230740/info/8b7b658c9d864b259c85da4f26b21bb9 2024-11-15T21:59:02,727 DEBUG [RS_CLOSE_META-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_CLOSE_META}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3be1b200-2a9c-651b-3f6b-ac964062b7d6/f4b02001-5f5c-c384-44d5-c547c79a7f1b/data/hbase/meta/1588230740/info/8b7b658c9d864b259c85da4f26b21bb9, delay=954 ms 2024-11-15T21:59:02,729 INFO [RS_CLOSE_META-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3be1b200-2a9c-651b-3f6b-ac964062b7d6/f4b02001-5f5c-c384-44d5-c547c79a7f1b/data/hbase/meta/1588230740/info/8b7b658c9d864b259c85da4f26b21bb9, entries=15, sequenceid=15, filesize=7.6 K 2024-11-15T21:59:02,729 DEBUG [RS_CLOSE_META-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3be1b200-2a9c-651b-3f6b-ac964062b7d6/f4b02001-5f5c-c384-44d5-c547c79a7f1b/data/hbase/meta/1588230740/.tmp/ns/547a163437dd4a3886ce4290badb63ae as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3be1b200-2a9c-651b-3f6b-ac964062b7d6/f4b02001-5f5c-c384-44d5-c547c79a7f1b/data/hbase/meta/1588230740/ns/547a163437dd4a3886ce4290badb63ae 2024-11-15T21:59:02,731 DEBUG [RS_CLOSE_META-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_CLOSE_META}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3be1b200-2a9c-651b-3f6b-ac964062b7d6/f4b02001-5f5c-c384-44d5-c547c79a7f1b/data/hbase/meta/1588230740/ns/547a163437dd4a3886ce4290badb63ae, delay=933 ms 2024-11-15T21:59:02,732 INFO [RS_CLOSE_META-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3be1b200-2a9c-651b-3f6b-ac964062b7d6/f4b02001-5f5c-c384-44d5-c547c79a7f1b/data/hbase/meta/1588230740/ns/547a163437dd4a3886ce4290badb63ae, entries=2, sequenceid=15, filesize=5.0 K 2024-11-15T21:59:02,733 DEBUG [RS_CLOSE_META-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3be1b200-2a9c-651b-3f6b-ac964062b7d6/f4b02001-5f5c-c384-44d5-c547c79a7f1b/data/hbase/meta/1588230740/.tmp/table/a7692679ab064b2384c9ffda13eb9965 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3be1b200-2a9c-651b-3f6b-ac964062b7d6/f4b02001-5f5c-c384-44d5-c547c79a7f1b/data/hbase/meta/1588230740/table/a7692679ab064b2384c9ffda13eb9965 2024-11-15T21:59:02,734 DEBUG [RS_CLOSE_META-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_CLOSE_META}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3be1b200-2a9c-651b-3f6b-ac964062b7d6/f4b02001-5f5c-c384-44d5-c547c79a7f1b/data/hbase/meta/1588230740/table/a7692679ab064b2384c9ffda13eb9965, delay=968 ms 2024-11-15T21:59:02,735 INFO [RS_CLOSE_META-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3be1b200-2a9c-651b-3f6b-ac964062b7d6/f4b02001-5f5c-c384-44d5-c547c79a7f1b/data/hbase/meta/1588230740/table/a7692679ab064b2384c9ffda13eb9965, entries=2, sequenceid=15, filesize=5.3 K 2024-11-15T21:59:02,736 INFO [RS_CLOSE_META-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~3.70 KB/3789, heapSize ~6.42 KB/6576, currentSize=0 B/0 for 1588230740 in 95ms, sequenceid=15, compaction requested=false 2024-11-15T21:59:02,736 DEBUG [RS_CLOSE_META-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_CLOSE_META}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta' 2024-11-15T21:59:02,738 DEBUG [StoreFileCloser-1588230740-info-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3be1b200-2a9c-651b-3f6b-ac964062b7d6/f4b02001-5f5c-c384-44d5-c547c79a7f1b/data/hbase/meta/1588230740/info/8b7b658c9d864b259c85da4f26b21bb9 2024-11-15T21:59:02,738 DEBUG [StoreFileCloser-1588230740-ns-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3be1b200-2a9c-651b-3f6b-ac964062b7d6/f4b02001-5f5c-c384-44d5-c547c79a7f1b/data/hbase/meta/1588230740/ns/547a163437dd4a3886ce4290badb63ae 2024-11-15T21:59:02,739 DEBUG [StoreFileCloser-1588230740-table-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3be1b200-2a9c-651b-3f6b-ac964062b7d6/f4b02001-5f5c-c384-44d5-c547c79a7f1b/data/hbase/meta/1588230740/table/a7692679ab064b2384c9ffda13eb9965 2024-11-15T21:59:02,744 DEBUG [RS_CLOSE_META-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3be1b200-2a9c-651b-3f6b-ac964062b7d6/f4b02001-5f5c-c384-44d5-c547c79a7f1b/data/hbase/meta/1588230740/recovered.edits/18.seqid, newMaxSeqId=18, maxSeqId=1 2024-11-15T21:59:02,744 DEBUG [RS_CLOSE_META-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-15T21:59:02,745 INFO [RS_CLOSE_META-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-15T21:59:02,745 DEBUG [RS_CLOSE_META-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1731707942641Running coprocessor pre-close hooks at 1731707942641Disabling compacts and flushes for region at 1731707942641Disabling writes for close at 1731707942641Obtaining lock to block concurrent updates at 1731707942642 (+1 ms)Preparing flush snapshotting stores in 1588230740 at 1731707942642Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=3789, getHeapSize=6816, getOffHeapSize=0, getCellsCount=27 at 1731707942643 (+1 ms)Flushing stores of hbase:meta,,1.1588230740 at 1731707942643Flushing 1588230740/info: creating writer at 1731707942643Flushing 1588230740/info: appending metadata at 1731707942672 (+29 ms)Flushing 1588230740/info: closing flushed file at 1731707942673 (+1 ms)Flushing 1588230740/ns: creating writer at 1731707942677 (+4 ms)Flushing 1588230740/ns: appending metadata at 1731707942698 (+21 ms)Flushing 1588230740/ns: closing flushed file at 1731707942698Flushing 1588230740/table: creating writer at 1731707942702 (+4 ms)Flushing 1588230740/table: appending metadata at 1731707942722 (+20 ms)Flushing 1588230740/table: closing flushed file at 1731707942722Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@21aab4d5: reopening flushed file at 1731707942726 (+4 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@74c70dd4: reopening flushed file at 1731707942729 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@6887cbcd: reopening flushed file at 1731707942732 (+3 ms)Finished flush of dataSize ~3.70 KB/3789, heapSize ~6.42 KB/6576, currentSize=0 B/0 for 1588230740 in 95ms, sequenceid=15, compaction requested=false at 1731707942736 (+4 ms)Writing region close event to WAL at 1731707942739 (+3 ms)Running coprocessor post-close hooks at 1731707942744 (+5 ms)Closed at 1731707942744 2024-11-15T21:59:02,745 DEBUG [RS_CLOSE_META-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-11-15T21:59:02,753 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35509-0x100f86912970001, quorum=127.0.0.1:62797, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-15T21:59:02,753 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35509-0x100f86912970001, quorum=127.0.0.1:62797, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-15T21:59:02,753 INFO [RS:0;ae3311ea5ccb:35509 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-15T21:59:02,754 INFO [RS:0;ae3311ea5ccb:35509 {}] regionserver.HRegionServer(1031): Exiting; stopping=ae3311ea5ccb,35509,1731707937420; zookeeper connection closed. 2024-11-15T21:59:02,754 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@56f80bff {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@56f80bff 2024-11-15T21:59:02,841 INFO [RS:1;ae3311ea5ccb:32843 {}] regionserver.HRegionServer(976): stopping server ae3311ea5ccb,32843,1731707937562; all regions closed. 2024-11-15T21:59:02,844 DEBUG [RS:1;ae3311ea5ccb:32843 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3be1b200-2a9c-651b-3f6b-ac964062b7d6/f4b02001-5f5c-c384-44d5-c547c79a7f1b/oldWALs 2024-11-15T21:59:02,844 INFO [RS:1;ae3311ea5ccb:32843 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL ae3311ea5ccb%2C32843%2C1731707937562.meta:.meta(num 1731707939800) 2024-11-15T21:59:02,846 DEBUG [RS:1;ae3311ea5ccb:32843 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3be1b200-2a9c-651b-3f6b-ac964062b7d6/f4b02001-5f5c-c384-44d5-c547c79a7f1b/oldWALs 2024-11-15T21:59:02,846 INFO [RS:1;ae3311ea5ccb:32843 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL ae3311ea5ccb%2C32843%2C1731707937562:(num 1731707939390) 2024-11-15T21:59:02,846 DEBUG [RS:1;ae3311ea5ccb:32843 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T21:59:02,846 INFO [RS:1;ae3311ea5ccb:32843 {}] regionserver.LeaseManager(133): Closed leases 2024-11-15T21:59:02,846 INFO [RS:1;ae3311ea5ccb:32843 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-15T21:59:02,846 INFO [RS:1;ae3311ea5ccb:32843 {}] hbase.ChoreService(370): Chore service for: regionserver/ae3311ea5ccb:0 had [ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-11-15T21:59:02,847 INFO [RS:1;ae3311ea5ccb:32843 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-15T21:59:02,847 INFO [regionserver/ae3311ea5ccb:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-15T21:59:02,847 INFO [RS:1;ae3311ea5ccb:32843 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.3:32843 2024-11-15T21:59:02,849 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:32843-0x100f86912970002, quorum=127.0.0.1:62797, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/ae3311ea5ccb,32843,1731707937562 2024-11-15T21:59:02,849 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40769-0x100f86912970000, quorum=127.0.0.1:62797, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-15T21:59:02,849 INFO [RS:1;ae3311ea5ccb:32843 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-15T21:59:02,851 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [ae3311ea5ccb,32843,1731707937562] 2024-11-15T21:59:02,853 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/ae3311ea5ccb,32843,1731707937562 already deleted, retry=false 2024-11-15T21:59:02,853 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; ae3311ea5ccb,32843,1731707937562 expired; onlineServers=0 2024-11-15T21:59:02,853 INFO [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master 'ae3311ea5ccb,40769,1731707936561' ***** 2024-11-15T21:59:02,853 INFO [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0 2024-11-15T21:59:02,853 INFO [M:0;ae3311ea5ccb:40769 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-15T21:59:02,853 INFO [M:0;ae3311ea5ccb:40769 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-15T21:59:02,853 DEBUG [M:0;ae3311ea5ccb:40769 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2024-11-15T21:59:02,854 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2024-11-15T21:59:02,854 DEBUG [M:0;ae3311ea5ccb:40769 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2024-11-15T21:59:02,854 DEBUG [master/ae3311ea5ccb:0:becomeActiveMaster-HFileCleaner.large.0-1731707939043 {}] cleaner.HFileCleaner(306): Exit Thread[master/ae3311ea5ccb:0:becomeActiveMaster-HFileCleaner.large.0-1731707939043,5,FailOnTimeoutGroup] 2024-11-15T21:59:02,854 DEBUG [master/ae3311ea5ccb:0:becomeActiveMaster-HFileCleaner.small.0-1731707939044 {}] cleaner.HFileCleaner(306): Exit Thread[master/ae3311ea5ccb:0:becomeActiveMaster-HFileCleaner.small.0-1731707939044,5,FailOnTimeoutGroup] 2024-11-15T21:59:02,854 INFO [M:0;ae3311ea5ccb:40769 {}] hbase.ChoreService(370): Chore service for: master/ae3311ea5ccb:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown 2024-11-15T21:59:02,854 INFO [M:0;ae3311ea5ccb:40769 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-15T21:59:02,854 DEBUG [M:0;ae3311ea5ccb:40769 {}] master.HMaster(1795): Stopping service threads 2024-11-15T21:59:02,854 INFO [M:0;ae3311ea5ccb:40769 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2024-11-15T21:59:02,854 INFO [M:0;ae3311ea5ccb:40769 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-15T21:59:02,855 INFO [M:0;ae3311ea5ccb:40769 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-15T21:59:02,855 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2024-11-15T21:59:02,856 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40769-0x100f86912970000, quorum=127.0.0.1:62797, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-11-15T21:59:02,856 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40769-0x100f86912970000, quorum=127.0.0.1:62797, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-15T21:59:02,856 DEBUG [M:0;ae3311ea5ccb:40769 {}] zookeeper.ZKUtil(347): master:40769-0x100f86912970000, quorum=127.0.0.1:62797, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error) 2024-11-15T21:59:02,856 WARN [M:0;ae3311ea5ccb:40769 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null 2024-11-15T21:59:02,857 INFO [M:0;ae3311ea5ccb:40769 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3be1b200-2a9c-651b-3f6b-ac964062b7d6/f4b02001-5f5c-c384-44d5-c547c79a7f1b/.lastflushedseqids 2024-11-15T21:59:02,869 INFO [M:0;ae3311ea5ccb:40769 {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-15T21:59:02,869 INFO [M:0;ae3311ea5ccb:40769 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2024-11-15T21:59:02,870 DEBUG [M:0;ae3311ea5ccb:40769 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-15T21:59:02,870 INFO [M:0;ae3311ea5ccb:40769 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-15T21:59:02,870 DEBUG [M:0;ae3311ea5ccb:40769 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-15T21:59:02,870 DEBUG [M:0;ae3311ea5ccb:40769 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-15T21:59:02,870 DEBUG [M:0;ae3311ea5ccb:40769 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-15T21:59:02,870 INFO [M:0;ae3311ea5ccb:40769 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=34.67 KB heapSize=43.02 KB 2024-11-15T21:59:02,885 DEBUG [M:0;ae3311ea5ccb:40769 {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 1913b92a81a84a93a9b2ae386203c005 2024-11-15T21:59:02,904 DEBUG [M:0;ae3311ea5ccb:40769 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3be1b200-2a9c-651b-3f6b-ac964062b7d6/f4b02001-5f5c-c384-44d5-c547c79a7f1b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/1913b92a81a84a93a9b2ae386203c005 is 82, key is hbase:meta,,1/info:regioninfo/1731707939871/Put/seqid=0 2024-11-15T21:59:02,905 INFO [M:0;ae3311ea5ccb:40769 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=504 B at sequenceid=84 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3be1b200-2a9c-651b-3f6b-ac964062b7d6/f4b02001-5f5c-c384-44d5-c547c79a7f1b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/1913b92a81a84a93a9b2ae386203c005 2024-11-15T21:59:02,916 DEBUG [M:0;ae3311ea5ccb:40769 {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on bb60597c8bf4498f88ad50ee60cacba7 2024-11-15T21:59:02,936 DEBUG [M:0;ae3311ea5ccb:40769 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3be1b200-2a9c-651b-3f6b-ac964062b7d6/f4b02001-5f5c-c384-44d5-c547c79a7f1b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/bb60597c8bf4498f88ad50ee60cacba7 is 778, key is \x00\x00\x00\x00\x00\x00\x00\x04/proc:d/1731707940648/Put/seqid=0 2024-11-15T21:59:02,937 INFO [M:0;ae3311ea5ccb:40769 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=34.05 KB at sequenceid=84 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3be1b200-2a9c-651b-3f6b-ac964062b7d6/f4b02001-5f5c-c384-44d5-c547c79a7f1b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/bb60597c8bf4498f88ad50ee60cacba7 2024-11-15T21:59:02,946 DEBUG [M:0;ae3311ea5ccb:40769 {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on a945182c941b4ea49d4b44afe9763386 2024-11-15T21:59:02,952 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:32843-0x100f86912970002, quorum=127.0.0.1:62797, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-15T21:59:02,952 INFO [RS:1;ae3311ea5ccb:32843 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-15T21:59:02,952 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:32843-0x100f86912970002, quorum=127.0.0.1:62797, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-15T21:59:02,952 INFO [RS:1;ae3311ea5ccb:32843 {}] regionserver.HRegionServer(1031): Exiting; stopping=ae3311ea5ccb,32843,1731707937562; zookeeper connection closed. 2024-11-15T21:59:02,952 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@73100617 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@73100617 2024-11-15T21:59:02,953 INFO [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 2 regionserver(s) complete 2024-11-15T21:59:02,959 DEBUG [M:0;ae3311ea5ccb:40769 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3be1b200-2a9c-651b-3f6b-ac964062b7d6/f4b02001-5f5c-c384-44d5-c547c79a7f1b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/a945182c941b4ea49d4b44afe9763386 is 69, key is ae3311ea5ccb,32843,1731707937562/rs:state/1731707939081/Put/seqid=0 2024-11-15T21:59:02,960 INFO [M:0;ae3311ea5ccb:40769 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=130 B at sequenceid=84 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3be1b200-2a9c-651b-3f6b-ac964062b7d6/f4b02001-5f5c-c384-44d5-c547c79a7f1b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/a945182c941b4ea49d4b44afe9763386 2024-11-15T21:59:02,962 DEBUG [M:0;ae3311ea5ccb:40769 {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3be1b200-2a9c-651b-3f6b-ac964062b7d6/f4b02001-5f5c-c384-44d5-c547c79a7f1b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/1913b92a81a84a93a9b2ae386203c005 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3be1b200-2a9c-651b-3f6b-ac964062b7d6/f4b02001-5f5c-c384-44d5-c547c79a7f1b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/1913b92a81a84a93a9b2ae386203c005 2024-11-15T21:59:02,964 INFO [M:0;ae3311ea5ccb:40769 {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3be1b200-2a9c-651b-3f6b-ac964062b7d6/f4b02001-5f5c-c384-44d5-c547c79a7f1b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/1913b92a81a84a93a9b2ae386203c005, entries=8, sequenceid=84, filesize=5.5 K 2024-11-15T21:59:02,964 DEBUG [M:0;ae3311ea5ccb:40769 {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3be1b200-2a9c-651b-3f6b-ac964062b7d6/f4b02001-5f5c-c384-44d5-c547c79a7f1b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/bb60597c8bf4498f88ad50ee60cacba7 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3be1b200-2a9c-651b-3f6b-ac964062b7d6/f4b02001-5f5c-c384-44d5-c547c79a7f1b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/bb60597c8bf4498f88ad50ee60cacba7 2024-11-15T21:59:02,966 INFO [M:0;ae3311ea5ccb:40769 {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3be1b200-2a9c-651b-3f6b-ac964062b7d6/f4b02001-5f5c-c384-44d5-c547c79a7f1b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/bb60597c8bf4498f88ad50ee60cacba7, entries=9, sequenceid=84, filesize=6.5 K 2024-11-15T21:59:02,967 DEBUG [M:0;ae3311ea5ccb:40769 {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3be1b200-2a9c-651b-3f6b-ac964062b7d6/f4b02001-5f5c-c384-44d5-c547c79a7f1b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/a945182c941b4ea49d4b44afe9763386 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3be1b200-2a9c-651b-3f6b-ac964062b7d6/f4b02001-5f5c-c384-44d5-c547c79a7f1b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/a945182c941b4ea49d4b44afe9763386 2024-11-15T21:59:02,968 INFO [M:0;ae3311ea5ccb:40769 {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3be1b200-2a9c-651b-3f6b-ac964062b7d6/f4b02001-5f5c-c384-44d5-c547c79a7f1b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/a945182c941b4ea49d4b44afe9763386, entries=2, sequenceid=84, filesize=5.1 K 2024-11-15T21:59:02,969 INFO [M:0;ae3311ea5ccb:40769 {}] regionserver.HRegion(3140): Finished flush of dataSize ~34.67 KB/35506, heapSize ~42.73 KB/43752, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 99ms, sequenceid=84, compaction requested=false 2024-11-15T21:59:02,973 INFO [M:0;ae3311ea5ccb:40769 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-15T21:59:02,973 DEBUG [M:0;ae3311ea5ccb:40769 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1731707942869Disabling compacts and flushes for region at 1731707942870 (+1 ms)Disabling writes for close at 1731707942870Obtaining lock to block concurrent updates at 1731707942870Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1731707942870Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=35506, getHeapSize=43992, getOffHeapSize=0, getCellsCount=99 at 1731707942870Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1731707942871 (+1 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1731707942871Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1731707942904 (+33 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1731707942904Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1731707942907 (+3 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1731707942935 (+28 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1731707942935Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1731707942939 (+4 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1731707942959 (+20 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1731707942959Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@6da7d62b: reopening flushed file at 1731707942962 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@3d8bcbb1: reopening flushed file at 1731707942964 (+2 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5fe5f1ef: reopening flushed file at 1731707942966 (+2 ms)Finished flush of dataSize ~34.67 KB/35506, heapSize ~42.73 KB/43752, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 99ms, sequenceid=84, compaction requested=false at 1731707942969 (+3 ms)Writing region close event to WAL at 1731707942973 (+4 ms)Closed at 1731707942973 2024-11-15T21:59:02,978 INFO [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-15T21:59:02,979 INFO [M:0;ae3311ea5ccb:40769 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down. 2024-11-15T21:59:02,979 INFO [M:0;ae3311ea5ccb:40769 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.3:40769 2024-11-15T21:59:02,979 INFO [M:0;ae3311ea5ccb:40769 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-15T21:59:03,082 INFO [M:0;ae3311ea5ccb:40769 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-15T21:59:03,082 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40769-0x100f86912970000, quorum=127.0.0.1:62797, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-15T21:59:03,082 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40769-0x100f86912970000, quorum=127.0.0.1:62797, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-15T21:59:03,087 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers 2024-11-15T21:59:03,107 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down 2024-11-15T21:59:03,108 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-15T21:59:03,116 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: io.hfile.TestBlockEvictionOnRegionMovement#testBlockEvictionOnRegionMove Thread=51 (was 12) Potentially hanging thread: master/ae3311ea5ccb:0:becomeActiveMaster-MemStoreChunkPool Statistics java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Timer for 'HBase' metrics system java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.util.TimerThread.mainLoop(Timer.java:563) java.base@17.0.11/java.util.TimerThread.run(Timer.java:516) Potentially hanging thread: regionserver/ae3311ea5ccb:0.procedureResultReporter java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.RemoteProcedureResultReporter.run(RemoteProcedureResultReporter.java:75) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-3-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: SnapshotHandlerChoreCleaner java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HMaster-EventLoopGroup-1-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: region-location-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1170) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-3-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: hfile-prefetch-1731707942727 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1177) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RpcClient-timer-pool-0 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.waitForNextTick(HashedWheelTimer.java:598) app//org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:494) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Idle-Rpc-Conn-Sweeper-pool-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-3-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-4-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Monitor thread for TaskMonitor java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hbase.monitoring.TaskMonitor$MonitorRunnable.run(TaskMonitor.java:325) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ForkJoinPool.commonPool-worker-3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.ForkJoinPool.awaitWork(ForkJoinPool.java:1724) java.base@17.0.11/java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1623) java.base@17.0.11/java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:165) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-4-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: weak-ref-cleaner-strictcontextstorage java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:155) java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:176) app//io.opentelemetry.context.StrictContextStorage$PendingScopes.run(StrictContextStorage.java:269) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: hfile-prefetch-1731707942558 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1177) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HBase-Metrics2-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Async-Client-Retry-Timer-pool-0 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.waitForNextTick(HashedWheelTimer.java:598) app//org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:494) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: master/ae3311ea5ccb:0:becomeActiveMaster-MemStoreChunkPool Statistics java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: hfile-prefetch-1731707941011 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: SessionTracker java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.zookeeper.server.SessionTrackerImpl.run(SessionTrackerImpl.java:163) Potentially hanging thread: Time-limited test.named-queue-events-pool-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) app//com.lmax.disruptor.BlockingWaitStrategy.waitFor(BlockingWaitStrategy.java:47) app//com.lmax.disruptor.ProcessingSequenceBarrier.waitFor(ProcessingSequenceBarrier.java:56) app//com.lmax.disruptor.BatchEventProcessor.processEvents(BatchEventProcessor.java:159) app//com.lmax.disruptor.BatchEventProcessor.run(BatchEventProcessor.java:125) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: regionserver/ae3311ea5ccb:0.procedureResultReporter java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.RemoteProcedureResultReporter.run(RemoteProcedureResultReporter.java:75) Potentially hanging thread: org.apache.hadoop.fs.FileSystem$Statistics$StatisticsDataReferenceCleaner java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:155) java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:176) app//org.apache.hadoop.fs.FileSystem$Statistics$StatisticsDataReferenceCleaner.run(FileSystem.java:4171) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HMaster-EventLoopGroup-1-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HMaster-EventLoopGroup-1-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-5 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-4 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-4-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ForkJoinPool.commonPool-worker-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.ForkJoinPool.awaitWork(ForkJoinPool.java:1724) java.base@17.0.11/java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1623) java.base@17.0.11/java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:165) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: hfile-prefetch-1731707942731 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1177) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ForkJoinPool.commonPool-worker-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkUntil(LockSupport.java:410) java.base@17.0.11/java.util.concurrent.ForkJoinPool.awaitWork(ForkJoinPool.java:1726) java.base@17.0.11/java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1623) java.base@17.0.11/java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:165) Potentially hanging thread: region-location-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1170) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=346 (was 287) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=939 (was 1083), ProcessCount=11 (was 11), AvailableMemoryMB=4181 (was 3331) - AvailableMemoryMB LEAK? - 2024-11-15T21:59:03,123 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: io.hfile.TestBlockEvictionOnRegionMovement#testBlockEvictionOnGracefulStop Thread=51, OpenFileDescriptor=346, MaxFileDescriptor=1048576, SystemLoadAverage=939, ProcessCount=11, AvailableMemoryMB=4180 2024-11-15T21:59:03,123 INFO [Time-limited test {}] hbase.HBaseTestingUtil(401): System.getProperty("hadoop.log.dir") already set to: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3be1b200-2a9c-651b-3f6b-ac964062b7d6/hadoop-log-dir so I do NOT create it in target/test-data/f706088d-a2ef-5fec-6a8f-fdcb2a34ea2f 2024-11-15T21:59:03,123 INFO [Time-limited test {}] hbase.HBaseTestingUtil(401): System.getProperty("hadoop.tmp.dir") already set to: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3be1b200-2a9c-651b-3f6b-ac964062b7d6/hadoop-tmp-dir so I do NOT create it in target/test-data/f706088d-a2ef-5fec-6a8f-fdcb2a34ea2f 2024-11-15T21:59:03,123 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-15T21:59:03,126 INFO [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f706088d-a2ef-5fec-6a8f-fdcb2a34ea2f/cluster_8cd5bc62-1883-a995-78d3-bbfda2509c45, deleteOnExit=true 2024-11-15T21:59:03,129 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f706088d-a2ef-5fec-6a8f-fdcb2a34ea2f/cluster_8cd5bc62-1883-a995-78d3-bbfda2509c45/zookeeper_0, clientPort=54544, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f706088d-a2ef-5fec-6a8f-fdcb2a34ea2f/cluster_8cd5bc62-1883-a995-78d3-bbfda2509c45/zookeeper_0/version-2, dataDirSize=457 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f706088d-a2ef-5fec-6a8f-fdcb2a34ea2f/cluster_8cd5bc62-1883-a995-78d3-bbfda2509c45/zookeeper_0/version-2, dataLogSize=457 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0 2024-11-15T21:59:03,130 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=54544 2024-11-15T21:59:03,143 INFO [Time-limited test {}] util.FSUtils(489): Created version file at /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3be1b200-2a9c-651b-3f6b-ac964062b7d6/f4b02001-5f5c-c384-44d5-c547c79a7f1b with version=8 2024-11-15T21:59:03,143 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1141): The hbase.fs.tmp.dir is set to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3be1b200-2a9c-651b-3f6b-ac964062b7d6/f4b02001-5f5c-c384-44d5-c547c79a7f1b/hbase-staging 2024-11-15T21:59:03,146 INFO [Time-limited test {}] client.ConnectionUtils(128): master/ae3311ea5ccb:0 server-side Connection retries=45 2024-11-15T21:59:03,146 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-15T21:59:03,147 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-15T21:59:03,147 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-15T21:59:03,147 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-15T21:59:03,147 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-15T21:59:03,147 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService 2024-11-15T21:59:03,147 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-15T21:59:03,148 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.3:42653 2024-11-15T21:59:03,149 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:42653 connecting to ZooKeeper ensemble=127.0.0.1:54544 2024-11-15T21:59:03,158 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:426530x0, quorum=127.0.0.1:54544, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-15T21:59:03,159 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:42653-0x100f8692d830000 connected 2024-11-15T21:59:03,177 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-15T21:59:03,178 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-15T21:59:03,179 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:42653-0x100f8692d830000, quorum=127.0.0.1:54544, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-15T21:59:03,179 INFO [Time-limited test {}] master.HMaster(525): hbase.rootdir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3be1b200-2a9c-651b-3f6b-ac964062b7d6/f4b02001-5f5c-c384-44d5-c547c79a7f1b, hbase.cluster.distributed=false 2024-11-15T21:59:03,181 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:42653-0x100f8692d830000, quorum=127.0.0.1:54544, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-15T21:59:03,183 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=42653 2024-11-15T21:59:03,183 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=42653 2024-11-15T21:59:03,183 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=42653 2024-11-15T21:59:03,183 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=42653 2024-11-15T21:59:03,183 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=42653 2024-11-15T21:59:03,200 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/ae3311ea5ccb:0 server-side Connection retries=45 2024-11-15T21:59:03,200 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-15T21:59:03,200 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-15T21:59:03,200 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-15T21:59:03,201 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-15T21:59:03,201 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-15T21:59:03,201 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-15T21:59:03,201 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-15T21:59:03,202 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.3:41593 2024-11-15T21:59:03,203 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:41593 connecting to ZooKeeper ensemble=127.0.0.1:54544 2024-11-15T21:59:03,203 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-15T21:59:03,203 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-15T21:59:03,207 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:415930x0, quorum=127.0.0.1:54544, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-15T21:59:03,207 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:41593-0x100f8692d830001 connected 2024-11-15T21:59:03,208 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:41593-0x100f8692d830001, quorum=127.0.0.1:54544, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-15T21:59:03,208 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-15T21:59:03,209 INFO [Time-limited test {}] bucket.FileIOEngine(99): Allocating cache 400 MB, on the path:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f706088d-a2ef-5fec-6a8f-fdcb2a34ea2f/bucket.cache 2024-11-15T21:59:03,209 INFO [Time-limited test {}] bucket.BucketCache(364): Instantiating BucketCache with acceptableFactor: 0.95, minFactor: 0.85, extraFreeFactor: 0.1, singleFactor: 0.25, multiFactor: 0.5, memoryFactor: 0.25, useStrongRef: false 2024-11-15T21:59:03,209 INFO [Time-limited test {}] bucket.BucketCache(478): Starting BucketCachePersisterThread 2024-11-15T21:59:03,209 INFO [Time-limited test {}] bucket.BucketCachePersister(38): BucketCachePersister started with interval: 100 2024-11-15T21:59:03,209 INFO [Thread-120 {}] bucket.BucketCache(1530): Started retrieving bucket cache from file 2024-11-15T21:59:03,209 INFO [Time-limited test {}] bucket.BucketCache(403): Started bucket cache; ioengine=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f706088d-a2ef-5fec-6a8f-fdcb2a34ea2f/bucket.cache, capacity=400 MB, blockSize=64 KB, writerThreadNum=3, writerQLen=64, persistencePath=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f706088d-a2ef-5fec-6a8f-fdcb2a34ea2f/bucket.persistence, bucketAllocator=org.apache.hadoop.hbase.io.hfile.bucket.BucketAllocator 2024-11-15T21:59:03,209 WARN [Thread-120 {}] bucket.BucketCache(1533): Persistence file missing! It's ok if it's first run after enabling persistent cache. 2024-11-15T21:59:03,210 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-15T21:59:03,210 INFO [Thread-120 {}] bucket.BucketAllocator(343): Cache totalSize=418148352, buckets=199, bucket capacity=2101248=(4*525312)=(FEWEST_ITEMS_IN_BUCKET*(largest configured bucketcache size)) 2024-11-15T21:59:03,210 INFO [Thread-120 {}] bucket.BucketCache(413): Persistent bucket cache recovery from /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f706088d-a2ef-5fec-6a8f-fdcb2a34ea2f/bucket.persistence is complete. 2024-11-15T21:59:03,211 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:41593-0x100f8692d830001, quorum=127.0.0.1:54544, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-15T21:59:03,212 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:41593-0x100f8692d830001, quorum=127.0.0.1:54544, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-15T21:59:03,212 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=41593 2024-11-15T21:59:03,212 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=41593 2024-11-15T21:59:03,213 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=41593 2024-11-15T21:59:03,216 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=41593 2024-11-15T21:59:03,217 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=41593 2024-11-15T21:59:03,241 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/ae3311ea5ccb:0 server-side Connection retries=45 2024-11-15T21:59:03,241 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-15T21:59:03,241 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-15T21:59:03,241 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-15T21:59:03,241 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-15T21:59:03,242 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-15T21:59:03,242 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-15T21:59:03,242 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-15T21:59:03,246 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.3:36535 2024-11-15T21:59:03,248 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:36535 connecting to ZooKeeper ensemble=127.0.0.1:54544 2024-11-15T21:59:03,248 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-15T21:59:03,248 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-15T21:59:03,252 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:365350x0, quorum=127.0.0.1:54544, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-15T21:59:03,253 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:365350x0, quorum=127.0.0.1:54544, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-15T21:59:03,253 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-15T21:59:03,257 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:36535-0x100f8692d830002 connected 2024-11-15T21:59:03,260 INFO [Time-limited test {}] bucket.FileIOEngine(99): Allocating cache 400 MB, on the path:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f706088d-a2ef-5fec-6a8f-fdcb2a34ea2f/bucket.cache 2024-11-15T21:59:03,260 INFO [Time-limited test {}] bucket.BucketCache(364): Instantiating BucketCache with acceptableFactor: 0.95, minFactor: 0.85, extraFreeFactor: 0.1, singleFactor: 0.25, multiFactor: 0.5, memoryFactor: 0.25, useStrongRef: false 2024-11-15T21:59:03,260 INFO [Time-limited test {}] bucket.BucketCache(478): Starting BucketCachePersisterThread 2024-11-15T21:59:03,260 INFO [Time-limited test {}] bucket.BucketCachePersister(38): BucketCachePersister started with interval: 100 2024-11-15T21:59:03,265 INFO [Thread-121 {}] bucket.BucketCache(1530): Started retrieving bucket cache from file 2024-11-15T21:59:03,265 WARN [Thread-121 {}] bucket.BucketCache(1533): Persistence file missing! It's ok if it's first run after enabling persistent cache. 2024-11-15T21:59:03,265 INFO [Time-limited test {}] bucket.BucketCache(403): Started bucket cache; ioengine=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f706088d-a2ef-5fec-6a8f-fdcb2a34ea2f/bucket.cache, capacity=400 MB, blockSize=64 KB, writerThreadNum=3, writerQLen=64, persistencePath=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f706088d-a2ef-5fec-6a8f-fdcb2a34ea2f/bucket.persistence, bucketAllocator=org.apache.hadoop.hbase.io.hfile.bucket.BucketAllocator 2024-11-15T21:59:03,265 INFO [Thread-121 {}] bucket.BucketAllocator(343): Cache totalSize=418148352, buckets=199, bucket capacity=2101248=(4*525312)=(FEWEST_ITEMS_IN_BUCKET*(largest configured bucketcache size)) 2024-11-15T21:59:03,265 INFO [Thread-121 {}] bucket.BucketCache(413): Persistent bucket cache recovery from /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f706088d-a2ef-5fec-6a8f-fdcb2a34ea2f/bucket.persistence is complete. 2024-11-15T21:59:03,265 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-15T21:59:03,266 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:36535-0x100f8692d830002, quorum=127.0.0.1:54544, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-15T21:59:03,268 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:36535-0x100f8692d830002, quorum=127.0.0.1:54544, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-15T21:59:03,280 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=36535 2024-11-15T21:59:03,281 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=36535 2024-11-15T21:59:03,284 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=36535 2024-11-15T21:59:03,285 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=36535 2024-11-15T21:59:03,291 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=36535 2024-11-15T21:59:03,306 DEBUG [M:0;ae3311ea5ccb:42653 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;ae3311ea5ccb:42653 2024-11-15T21:59:03,308 INFO [master/ae3311ea5ccb:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/ae3311ea5ccb,42653,1731707943146 2024-11-15T21:59:03,311 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42653-0x100f8692d830000, quorum=127.0.0.1:54544, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-15T21:59:03,311 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41593-0x100f8692d830001, quorum=127.0.0.1:54544, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-15T21:59:03,311 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36535-0x100f8692d830002, quorum=127.0.0.1:54544, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-15T21:59:03,313 DEBUG [master/ae3311ea5ccb:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:42653-0x100f8692d830000, quorum=127.0.0.1:54544, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/ae3311ea5ccb,42653,1731707943146 2024-11-15T21:59:03,315 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41593-0x100f8692d830001, quorum=127.0.0.1:54544, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-15T21:59:03,315 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41593-0x100f8692d830001, quorum=127.0.0.1:54544, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-15T21:59:03,315 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42653-0x100f8692d830000, quorum=127.0.0.1:54544, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-15T21:59:03,315 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36535-0x100f8692d830002, quorum=127.0.0.1:54544, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-15T21:59:03,315 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36535-0x100f8692d830002, quorum=127.0.0.1:54544, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-15T21:59:03,316 DEBUG [master/ae3311ea5ccb:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:42653-0x100f8692d830000, quorum=127.0.0.1:54544, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-11-15T21:59:03,317 INFO [master/ae3311ea5ccb:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/ae3311ea5ccb,42653,1731707943146 from backup master directory 2024-11-15T21:59:03,319 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36535-0x100f8692d830002, quorum=127.0.0.1:54544, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-15T21:59:03,319 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42653-0x100f8692d830000, quorum=127.0.0.1:54544, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/ae3311ea5ccb,42653,1731707943146 2024-11-15T21:59:03,319 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41593-0x100f8692d830001, quorum=127.0.0.1:54544, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-15T21:59:03,319 WARN [master/ae3311ea5ccb:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-15T21:59:03,319 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42653-0x100f8692d830000, quorum=127.0.0.1:54544, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-15T21:59:03,319 INFO [master/ae3311ea5ccb:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=ae3311ea5ccb,42653,1731707943146 2024-11-15T21:59:03,321 DEBUG [master/ae3311ea5ccb:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3be1b200-2a9c-651b-3f6b-ac964062b7d6/f4b02001-5f5c-c384-44d5-c547c79a7f1b/hbase.id] with ID: 25790b20-82e5-471e-ad18-1eac11333487 2024-11-15T21:59:03,321 DEBUG [master/ae3311ea5ccb:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3be1b200-2a9c-651b-3f6b-ac964062b7d6/f4b02001-5f5c-c384-44d5-c547c79a7f1b/.tmp/hbase.id 2024-11-15T21:59:03,342 DEBUG [master/ae3311ea5ccb:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3be1b200-2a9c-651b-3f6b-ac964062b7d6/f4b02001-5f5c-c384-44d5-c547c79a7f1b/.tmp/hbase.id]:[file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3be1b200-2a9c-651b-3f6b-ac964062b7d6/f4b02001-5f5c-c384-44d5-c547c79a7f1b/hbase.id] 2024-11-15T21:59:03,402 DEBUG [master/ae3311ea5ccb:0:becomeActiveMaster {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-15T21:59:03,402 INFO [master/ae3311ea5ccb:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem. 2024-11-15T21:59:03,405 INFO [master/ae3311ea5ccb:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 3ms. 2024-11-15T21:59:03,408 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42653-0x100f8692d830000, quorum=127.0.0.1:54544, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-15T21:59:03,408 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41593-0x100f8692d830001, quorum=127.0.0.1:54544, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-15T21:59:03,408 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36535-0x100f8692d830002, quorum=127.0.0.1:54544, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-15T21:59:03,413 INFO [master/ae3311ea5ccb:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-15T21:59:03,414 INFO [master/ae3311ea5ccb:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-11-15T21:59:03,415 INFO [master/ae3311ea5ccb:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-15T21:59:03,441 INFO [master/ae3311ea5ccb:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3be1b200-2a9c-651b-3f6b-ac964062b7d6/f4b02001-5f5c-c384-44d5-c547c79a7f1b/MasterData/data/master/store 2024-11-15T21:59:03,454 DEBUG [master/ae3311ea5ccb:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-15T21:59:03,454 DEBUG [master/ae3311ea5ccb:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-15T21:59:03,454 INFO [master/ae3311ea5ccb:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-15T21:59:03,454 DEBUG [master/ae3311ea5ccb:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-15T21:59:03,454 DEBUG [master/ae3311ea5ccb:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-15T21:59:03,454 DEBUG [master/ae3311ea5ccb:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-15T21:59:03,455 INFO [master/ae3311ea5ccb:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-15T21:59:03,455 DEBUG [master/ae3311ea5ccb:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1731707943454Disabling compacts and flushes for region at 1731707943454Disabling writes for close at 1731707943454Writing region close event to WAL at 1731707943454Closed at 1731707943454 2024-11-15T21:59:03,455 WARN [master/ae3311ea5ccb:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3be1b200-2a9c-651b-3f6b-ac964062b7d6/f4b02001-5f5c-c384-44d5-c547c79a7f1b/MasterData/data/master/store/.initializing 2024-11-15T21:59:03,455 DEBUG [master/ae3311ea5ccb:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3be1b200-2a9c-651b-3f6b-ac964062b7d6/f4b02001-5f5c-c384-44d5-c547c79a7f1b/MasterData/WALs/ae3311ea5ccb,42653,1731707943146 2024-11-15T21:59:03,470 INFO [master/ae3311ea5ccb:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-15T21:59:03,475 INFO [master/ae3311ea5ccb:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=ae3311ea5ccb%2C42653%2C1731707943146, suffix=, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3be1b200-2a9c-651b-3f6b-ac964062b7d6/f4b02001-5f5c-c384-44d5-c547c79a7f1b/MasterData/WALs/ae3311ea5ccb,42653,1731707943146, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3be1b200-2a9c-651b-3f6b-ac964062b7d6/f4b02001-5f5c-c384-44d5-c547c79a7f1b/MasterData/oldWALs, maxLogs=10 2024-11-15T21:59:03,503 INFO [master/ae3311ea5ccb:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3be1b200-2a9c-651b-3f6b-ac964062b7d6/f4b02001-5f5c-c384-44d5-c547c79a7f1b/MasterData/WALs/ae3311ea5ccb,42653,1731707943146/ae3311ea5ccb%2C42653%2C1731707943146.1731707943476 2024-11-15T21:59:03,503 DEBUG [master/ae3311ea5ccb:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-11-15T21:59:03,503 DEBUG [master/ae3311ea5ccb:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-11-15T21:59:03,503 DEBUG [master/ae3311ea5ccb:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-15T21:59:03,504 DEBUG [master/ae3311ea5ccb:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-11-15T21:59:03,504 DEBUG [master/ae3311ea5ccb:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-11-15T21:59:03,504 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-11-15T21:59:03,515 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-11-15T21:59:03,515 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T21:59:03,515 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-15T21:59:03,515 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-11-15T21:59:03,521 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-11-15T21:59:03,521 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T21:59:03,521 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-15T21:59:03,522 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-11-15T21:59:03,527 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-11-15T21:59:03,527 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T21:59:03,527 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-15T21:59:03,527 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-11-15T21:59:03,531 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-11-15T21:59:03,531 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T21:59:03,532 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-15T21:59:03,532 DEBUG [master/ae3311ea5ccb:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682 2024-11-15T21:59:03,533 DEBUG [master/ae3311ea5ccb:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3be1b200-2a9c-651b-3f6b-ac964062b7d6/f4b02001-5f5c-c384-44d5-c547c79a7f1b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-15T21:59:03,533 DEBUG [master/ae3311ea5ccb:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3be1b200-2a9c-651b-3f6b-ac964062b7d6/f4b02001-5f5c-c384-44d5-c547c79a7f1b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-15T21:59:03,533 DEBUG [master/ae3311ea5ccb:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682 2024-11-15T21:59:03,533 DEBUG [master/ae3311ea5ccb:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682 2024-11-15T21:59:03,534 DEBUG [master/ae3311ea5ccb:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-15T21:59:03,534 DEBUG [master/ae3311ea5ccb:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-11-15T21:59:03,560 DEBUG [master/ae3311ea5ccb:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3be1b200-2a9c-651b-3f6b-ac964062b7d6/f4b02001-5f5c-c384-44d5-c547c79a7f1b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-15T21:59:03,560 INFO [master/ae3311ea5ccb:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=68074014, jitterRate=0.014381855726242065}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-15T21:59:03,564 DEBUG [master/ae3311ea5ccb:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682: Writing region info on filesystem at 1731707943504Initializing all the Stores at 1731707943504Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731707943504Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731707943504Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731707943504Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731707943504Cleaning up temporary data from old regions at 1731707943533 (+29 ms)Region opened successfully at 1731707943564 (+31 ms) 2024-11-15T21:59:03,565 INFO [master/ae3311ea5ccb:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-11-15T21:59:03,570 DEBUG [master/ae3311ea5ccb:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7f252b44, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=ae3311ea5ccb/172.17.0.3:0 2024-11-15T21:59:03,571 INFO [master/ae3311ea5ccb:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating... 2024-11-15T21:59:03,572 INFO [master/ae3311ea5ccb:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5 2024-11-15T21:59:03,572 INFO [master/ae3311ea5ccb:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50 2024-11-15T21:59:03,572 INFO [master/ae3311ea5ccb:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-15T21:59:03,572 INFO [master/ae3311ea5ccb:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-15T21:59:03,573 INFO [master/ae3311ea5ccb:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 0 msec 2024-11-15T21:59:03,573 INFO [master/ae3311ea5ccb:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-11-15T21:59:03,579 INFO [master/ae3311ea5ccb:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2024-11-15T21:59:03,580 DEBUG [master/ae3311ea5ccb:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:42653-0x100f8692d830000, quorum=127.0.0.1:54544, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error) 2024-11-15T21:59:03,581 DEBUG [master/ae3311ea5ccb:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false 2024-11-15T21:59:03,582 INFO [master/ae3311ea5ccb:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2024-11-15T21:59:03,583 DEBUG [master/ae3311ea5ccb:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:42653-0x100f8692d830000, quorum=127.0.0.1:54544, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error) 2024-11-15T21:59:03,585 DEBUG [master/ae3311ea5ccb:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false 2024-11-15T21:59:03,585 INFO [master/ae3311ea5ccb:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2024-11-15T21:59:03,587 DEBUG [master/ae3311ea5ccb:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:42653-0x100f8692d830000, quorum=127.0.0.1:54544, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error) 2024-11-15T21:59:03,589 DEBUG [master/ae3311ea5ccb:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false 2024-11-15T21:59:03,594 DEBUG [master/ae3311ea5ccb:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:42653-0x100f8692d830000, quorum=127.0.0.1:54544, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error) 2024-11-15T21:59:03,595 DEBUG [master/ae3311ea5ccb:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false 2024-11-15T21:59:03,599 DEBUG [master/ae3311ea5ccb:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:42653-0x100f8692d830000, quorum=127.0.0.1:54544, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error) 2024-11-15T21:59:03,602 DEBUG [master/ae3311ea5ccb:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false 2024-11-15T21:59:03,605 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41593-0x100f8692d830001, quorum=127.0.0.1:54544, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-15T21:59:03,605 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41593-0x100f8692d830001, quorum=127.0.0.1:54544, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-15T21:59:03,605 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42653-0x100f8692d830000, quorum=127.0.0.1:54544, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-15T21:59:03,605 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36535-0x100f8692d830002, quorum=127.0.0.1:54544, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-15T21:59:03,605 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42653-0x100f8692d830000, quorum=127.0.0.1:54544, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-15T21:59:03,605 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36535-0x100f8692d830002, quorum=127.0.0.1:54544, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-15T21:59:03,607 INFO [master/ae3311ea5ccb:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=ae3311ea5ccb,42653,1731707943146, sessionid=0x100f8692d830000, setting cluster-up flag (Was=false) 2024-11-15T21:59:03,613 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36535-0x100f8692d830002, quorum=127.0.0.1:54544, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-15T21:59:03,613 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41593-0x100f8692d830001, quorum=127.0.0.1:54544, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-15T21:59:03,613 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42653-0x100f8692d830000, quorum=127.0.0.1:54544, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-15T21:59:03,621 DEBUG [master/ae3311ea5ccb:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort 2024-11-15T21:59:03,623 DEBUG [master/ae3311ea5ccb:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=ae3311ea5ccb,42653,1731707943146 2024-11-15T21:59:03,627 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36535-0x100f8692d830002, quorum=127.0.0.1:54544, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-15T21:59:03,627 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42653-0x100f8692d830000, quorum=127.0.0.1:54544, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-15T21:59:03,627 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41593-0x100f8692d830001, quorum=127.0.0.1:54544, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-15T21:59:03,635 DEBUG [master/ae3311ea5ccb:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort 2024-11-15T21:59:03,637 DEBUG [master/ae3311ea5ccb:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=ae3311ea5ccb,42653,1731707943146 2024-11-15T21:59:03,638 INFO [master/ae3311ea5ccb:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3be1b200-2a9c-651b-3f6b-ac964062b7d6/f4b02001-5f5c-c384-44d5-c547c79a7f1b/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again 2024-11-15T21:59:03,640 DEBUG [master/ae3311ea5ccb:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-15T21:59:03,640 INFO [master/ae3311ea5ccb:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2 2024-11-15T21:59:03,641 INFO [master/ae3311ea5ccb:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2024-11-15T21:59:03,641 DEBUG [master/ae3311ea5ccb:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: ae3311ea5ccb,42653,1731707943146 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2024-11-15T21:59:03,643 DEBUG [master/ae3311ea5ccb:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/ae3311ea5ccb:0, corePoolSize=5, maxPoolSize=5 2024-11-15T21:59:03,643 DEBUG [master/ae3311ea5ccb:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/ae3311ea5ccb:0, corePoolSize=5, maxPoolSize=5 2024-11-15T21:59:03,643 DEBUG [master/ae3311ea5ccb:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/ae3311ea5ccb:0, corePoolSize=5, maxPoolSize=5 2024-11-15T21:59:03,644 DEBUG [master/ae3311ea5ccb:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/ae3311ea5ccb:0, corePoolSize=5, maxPoolSize=5 2024-11-15T21:59:03,644 DEBUG [master/ae3311ea5ccb:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/ae3311ea5ccb:0, corePoolSize=10, maxPoolSize=10 2024-11-15T21:59:03,644 DEBUG [master/ae3311ea5ccb:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/ae3311ea5ccb:0, corePoolSize=1, maxPoolSize=1 2024-11-15T21:59:03,644 DEBUG [master/ae3311ea5ccb:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/ae3311ea5ccb:0, corePoolSize=2, maxPoolSize=2 2024-11-15T21:59:03,644 DEBUG [master/ae3311ea5ccb:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/ae3311ea5ccb:0, corePoolSize=1, maxPoolSize=1 2024-11-15T21:59:03,650 INFO [master/ae3311ea5ccb:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731707973649 2024-11-15T21:59:03,650 INFO [master/ae3311ea5ccb:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2024-11-15T21:59:03,650 INFO [master/ae3311ea5ccb:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2024-11-15T21:59:03,650 INFO [master/ae3311ea5ccb:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2024-11-15T21:59:03,650 INFO [master/ae3311ea5ccb:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2024-11-15T21:59:03,650 INFO [master/ae3311ea5ccb:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2024-11-15T21:59:03,650 INFO [master/ae3311ea5ccb:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2024-11-15T21:59:03,650 INFO [master/ae3311ea5ccb:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-15T21:59:03,651 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-15T21:59:03,651 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region 2024-11-15T21:59:03,651 INFO [master/ae3311ea5ccb:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2024-11-15T21:59:03,651 INFO [master/ae3311ea5ccb:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2024-11-15T21:59:03,651 INFO [master/ae3311ea5ccb:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2024-11-15T21:59:03,652 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T21:59:03,652 INFO [master/ae3311ea5ccb:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2024-11-15T21:59:03,652 INFO [master/ae3311ea5ccb:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2024-11-15T21:59:03,652 INFO [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-11-15T21:59:03,652 DEBUG [master/ae3311ea5ccb:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/ae3311ea5ccb:0:becomeActiveMaster-HFileCleaner.large.0-1731707943652,5,FailOnTimeoutGroup] 2024-11-15T21:59:03,653 DEBUG [master/ae3311ea5ccb:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/ae3311ea5ccb:0:becomeActiveMaster-HFileCleaner.small.0-1731707943652,5,FailOnTimeoutGroup] 2024-11-15T21:59:03,653 INFO [master/ae3311ea5ccb:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-15T21:59:03,653 INFO [master/ae3311ea5ccb:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2024-11-15T21:59:03,653 INFO [master/ae3311ea5ccb:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2024-11-15T21:59:03,653 INFO [master/ae3311ea5ccb:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2024-11-15T21:59:03,670 INFO [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3be1b200-2a9c-651b-3f6b-ac964062b7d6/f4b02001-5f5c-c384-44d5-c547c79a7f1b/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321 2024-11-15T21:59:03,671 INFO [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3be1b200-2a9c-651b-3f6b-ac964062b7d6/f4b02001-5f5c-c384-44d5-c547c79a7f1b 2024-11-15T21:59:03,678 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-15T21:59:03,679 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family info of region 1588230740 2024-11-15T21:59:03,682 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-15T21:59:03,682 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T21:59:03,683 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-15T21:59:03,683 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family ns of region 1588230740 2024-11-15T21:59:03,686 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-15T21:59:03,686 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T21:59:03,687 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-15T21:59:03,687 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family rep_barrier of region 1588230740 2024-11-15T21:59:03,691 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-15T21:59:03,691 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T21:59:03,691 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-15T21:59:03,691 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family table of region 1588230740 2024-11-15T21:59:03,694 INFO [RS:0;ae3311ea5ccb:41593 {}] regionserver.HRegionServer(746): ClusterId : 25790b20-82e5-471e-ad18-1eac11333487 2024-11-15T21:59:03,694 DEBUG [RS:0;ae3311ea5ccb:41593 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-15T21:59:03,695 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-15T21:59:03,696 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T21:59:03,696 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-15T21:59:03,696 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-15T21:59:03,696 INFO [RS:1;ae3311ea5ccb:36535 {}] regionserver.HRegionServer(746): ClusterId : 25790b20-82e5-471e-ad18-1eac11333487 2024-11-15T21:59:03,696 DEBUG [RS:1;ae3311ea5ccb:36535 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-15T21:59:03,697 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3be1b200-2a9c-651b-3f6b-ac964062b7d6/f4b02001-5f5c-c384-44d5-c547c79a7f1b/data/hbase/meta/1588230740 2024-11-15T21:59:03,697 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3be1b200-2a9c-651b-3f6b-ac964062b7d6/f4b02001-5f5c-c384-44d5-c547c79a7f1b/data/hbase/meta/1588230740 2024-11-15T21:59:03,697 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-15T21:59:03,697 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-15T21:59:03,698 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-15T21:59:03,698 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-15T21:59:03,700 DEBUG [RS:0;ae3311ea5ccb:41593 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-15T21:59:03,700 DEBUG [RS:0;ae3311ea5ccb:41593 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-15T21:59:03,703 DEBUG [RS:1;ae3311ea5ccb:36535 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-15T21:59:03,703 DEBUG [RS:1;ae3311ea5ccb:36535 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-15T21:59:03,705 DEBUG [RS:0;ae3311ea5ccb:41593 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-15T21:59:03,705 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3be1b200-2a9c-651b-3f6b-ac964062b7d6/f4b02001-5f5c-c384-44d5-c547c79a7f1b/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-15T21:59:03,706 DEBUG [RS:0;ae3311ea5ccb:41593 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3fde1ffe, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=ae3311ea5ccb/172.17.0.3:0 2024-11-15T21:59:03,706 INFO [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=63448428, jitterRate=-0.05454474687576294}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-15T21:59:03,708 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740: Writing region info on filesystem at 1731707943678Initializing all the Stores at 1731707943678Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731707943678Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731707943679 (+1 ms)Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731707943679Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731707943679Cleaning up temporary data from old regions at 1731707943697 (+18 ms)Region opened successfully at 1731707943707 (+10 ms) 2024-11-15T21:59:03,708 DEBUG [RS:1;ae3311ea5ccb:36535 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-15T21:59:03,708 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-15T21:59:03,708 INFO [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-15T21:59:03,708 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-15T21:59:03,708 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-15T21:59:03,708 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-15T21:59:03,708 DEBUG [RS:1;ae3311ea5ccb:36535 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1183ecfc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=ae3311ea5ccb/172.17.0.3:0 2024-11-15T21:59:03,709 INFO [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-15T21:59:03,710 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1731707943708Disabling compacts and flushes for region at 1731707943708Disabling writes for close at 1731707943708Writing region close event to WAL at 1731707943709 (+1 ms)Closed at 1731707943709 2024-11-15T21:59:03,711 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-15T21:59:03,711 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta 2024-11-15T21:59:03,711 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-11-15T21:59:03,712 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-15T21:59:03,714 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2024-11-15T21:59:03,723 DEBUG [RS:0;ae3311ea5ccb:41593 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;ae3311ea5ccb:41593 2024-11-15T21:59:03,723 INFO [RS:0;ae3311ea5ccb:41593 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-15T21:59:03,723 INFO [RS:0;ae3311ea5ccb:41593 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-15T21:59:03,723 DEBUG [RS:0;ae3311ea5ccb:41593 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-15T21:59:03,725 INFO [RS:0;ae3311ea5ccb:41593 {}] regionserver.HRegionServer(2659): reportForDuty to master=ae3311ea5ccb,42653,1731707943146 with port=41593, startcode=1731707943200 2024-11-15T21:59:03,725 DEBUG [RS:0;ae3311ea5ccb:41593 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-15T21:59:03,728 INFO [HMaster-EventLoopGroup-6-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:34171, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=RegionServerStatusService 2024-11-15T21:59:03,729 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=42653 {}] master.ServerManager(363): Checking decommissioned status of RegionServer ae3311ea5ccb,41593,1731707943200 2024-11-15T21:59:03,729 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=42653 {}] master.ServerManager(517): Registering regionserver=ae3311ea5ccb,41593,1731707943200 2024-11-15T21:59:03,731 DEBUG [RS:0;ae3311ea5ccb:41593 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3be1b200-2a9c-651b-3f6b-ac964062b7d6/f4b02001-5f5c-c384-44d5-c547c79a7f1b 2024-11-15T21:59:03,731 DEBUG [RS:0;ae3311ea5ccb:41593 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=file:/ 2024-11-15T21:59:03,731 DEBUG [RS:0;ae3311ea5ccb:41593 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-15T21:59:03,733 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42653-0x100f8692d830000, quorum=127.0.0.1:54544, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-15T21:59:03,734 DEBUG [RS:0;ae3311ea5ccb:41593 {}] zookeeper.ZKUtil(111): regionserver:41593-0x100f8692d830001, quorum=127.0.0.1:54544, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/ae3311ea5ccb,41593,1731707943200 2024-11-15T21:59:03,734 WARN [RS:0;ae3311ea5ccb:41593 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-15T21:59:03,734 INFO [RS:0;ae3311ea5ccb:41593 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-15T21:59:03,734 DEBUG [RS:0;ae3311ea5ccb:41593 {}] regionserver.HRegionServer(1793): logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3be1b200-2a9c-651b-3f6b-ac964062b7d6/f4b02001-5f5c-c384-44d5-c547c79a7f1b/WALs/ae3311ea5ccb,41593,1731707943200 2024-11-15T21:59:03,734 DEBUG [RS:1;ae3311ea5ccb:36535 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:1;ae3311ea5ccb:36535 2024-11-15T21:59:03,734 INFO [RS:1;ae3311ea5ccb:36535 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-15T21:59:03,734 INFO [RS:1;ae3311ea5ccb:36535 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-15T21:59:03,734 DEBUG [RS:1;ae3311ea5ccb:36535 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-15T21:59:03,736 INFO [RS:1;ae3311ea5ccb:36535 {}] regionserver.HRegionServer(2659): reportForDuty to master=ae3311ea5ccb,42653,1731707943146 with port=36535, startcode=1731707943240 2024-11-15T21:59:03,736 DEBUG [RS:1;ae3311ea5ccb:36535 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-15T21:59:03,740 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [ae3311ea5ccb,41593,1731707943200] 2024-11-15T21:59:03,743 INFO [HMaster-EventLoopGroup-6-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:46947, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=RegionServerStatusService 2024-11-15T21:59:03,743 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=42653 {}] master.ServerManager(363): Checking decommissioned status of RegionServer ae3311ea5ccb,36535,1731707943240 2024-11-15T21:59:03,743 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=42653 {}] master.ServerManager(517): Registering regionserver=ae3311ea5ccb,36535,1731707943240 2024-11-15T21:59:03,745 DEBUG [RS:1;ae3311ea5ccb:36535 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3be1b200-2a9c-651b-3f6b-ac964062b7d6/f4b02001-5f5c-c384-44d5-c547c79a7f1b 2024-11-15T21:59:03,745 DEBUG [RS:1;ae3311ea5ccb:36535 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=file:/ 2024-11-15T21:59:03,745 DEBUG [RS:1;ae3311ea5ccb:36535 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-15T21:59:03,753 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42653-0x100f8692d830000, quorum=127.0.0.1:54544, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-15T21:59:03,765 DEBUG [RS:1;ae3311ea5ccb:36535 {}] zookeeper.ZKUtil(111): regionserver:36535-0x100f8692d830002, quorum=127.0.0.1:54544, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/ae3311ea5ccb,36535,1731707943240 2024-11-15T21:59:03,765 WARN [RS:1;ae3311ea5ccb:36535 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-15T21:59:03,765 INFO [RS:1;ae3311ea5ccb:36535 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-15T21:59:03,765 DEBUG [RS:1;ae3311ea5ccb:36535 {}] regionserver.HRegionServer(1793): logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3be1b200-2a9c-651b-3f6b-ac964062b7d6/f4b02001-5f5c-c384-44d5-c547c79a7f1b/WALs/ae3311ea5ccb,36535,1731707943240 2024-11-15T21:59:03,765 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [ae3311ea5ccb,36535,1731707943240] 2024-11-15T21:59:03,767 INFO [RS:0;ae3311ea5ccb:41593 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-15T21:59:03,777 INFO [RS:0;ae3311ea5ccb:41593 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-15T21:59:03,780 INFO [RS:0;ae3311ea5ccb:41593 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-15T21:59:03,780 INFO [RS:0;ae3311ea5ccb:41593 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-15T21:59:03,781 INFO [RS:0;ae3311ea5ccb:41593 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-15T21:59:03,782 INFO [RS:0;ae3311ea5ccb:41593 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-15T21:59:03,782 INFO [RS:0;ae3311ea5ccb:41593 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-15T21:59:03,782 DEBUG [RS:0;ae3311ea5ccb:41593 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/ae3311ea5ccb:0, corePoolSize=1, maxPoolSize=1 2024-11-15T21:59:03,783 DEBUG [RS:0;ae3311ea5ccb:41593 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/ae3311ea5ccb:0, corePoolSize=1, maxPoolSize=1 2024-11-15T21:59:03,783 DEBUG [RS:0;ae3311ea5ccb:41593 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/ae3311ea5ccb:0, corePoolSize=1, maxPoolSize=1 2024-11-15T21:59:03,783 DEBUG [RS:0;ae3311ea5ccb:41593 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/ae3311ea5ccb:0, corePoolSize=1, maxPoolSize=1 2024-11-15T21:59:03,783 DEBUG [RS:0;ae3311ea5ccb:41593 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/ae3311ea5ccb:0, corePoolSize=1, maxPoolSize=1 2024-11-15T21:59:03,783 DEBUG [RS:0;ae3311ea5ccb:41593 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/ae3311ea5ccb:0, corePoolSize=2, maxPoolSize=2 2024-11-15T21:59:03,783 DEBUG [RS:0;ae3311ea5ccb:41593 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/ae3311ea5ccb:0, corePoolSize=1, maxPoolSize=1 2024-11-15T21:59:03,783 DEBUG [RS:0;ae3311ea5ccb:41593 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/ae3311ea5ccb:0, corePoolSize=1, maxPoolSize=1 2024-11-15T21:59:03,783 DEBUG [RS:0;ae3311ea5ccb:41593 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/ae3311ea5ccb:0, corePoolSize=1, maxPoolSize=1 2024-11-15T21:59:03,783 DEBUG [RS:0;ae3311ea5ccb:41593 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/ae3311ea5ccb:0, corePoolSize=1, maxPoolSize=1 2024-11-15T21:59:03,784 DEBUG [RS:0;ae3311ea5ccb:41593 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/ae3311ea5ccb:0, corePoolSize=1, maxPoolSize=1 2024-11-15T21:59:03,784 DEBUG [RS:0;ae3311ea5ccb:41593 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/ae3311ea5ccb:0, corePoolSize=1, maxPoolSize=1 2024-11-15T21:59:03,784 DEBUG [RS:0;ae3311ea5ccb:41593 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/ae3311ea5ccb:0, corePoolSize=3, maxPoolSize=3 2024-11-15T21:59:03,784 DEBUG [RS:0;ae3311ea5ccb:41593 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0, corePoolSize=3, maxPoolSize=3 2024-11-15T21:59:03,786 INFO [RS:0;ae3311ea5ccb:41593 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-15T21:59:03,786 INFO [RS:0;ae3311ea5ccb:41593 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-15T21:59:03,786 INFO [RS:0;ae3311ea5ccb:41593 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-15T21:59:03,786 INFO [RS:0;ae3311ea5ccb:41593 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-15T21:59:03,786 INFO [RS:0;ae3311ea5ccb:41593 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-15T21:59:03,786 INFO [RS:0;ae3311ea5ccb:41593 {}] hbase.ChoreService(168): Chore ScheduledChore name=ae3311ea5ccb,41593,1731707943200-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-15T21:59:03,790 INFO [RS:1;ae3311ea5ccb:36535 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-15T21:59:03,793 INFO [RS:1;ae3311ea5ccb:36535 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-15T21:59:03,796 INFO [RS:1;ae3311ea5ccb:36535 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-15T21:59:03,796 INFO [RS:1;ae3311ea5ccb:36535 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-15T21:59:03,796 INFO [RS:1;ae3311ea5ccb:36535 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-15T21:59:03,797 INFO [RS:1;ae3311ea5ccb:36535 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-15T21:59:03,797 INFO [RS:1;ae3311ea5ccb:36535 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-15T21:59:03,797 DEBUG [RS:1;ae3311ea5ccb:36535 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/ae3311ea5ccb:0, corePoolSize=1, maxPoolSize=1 2024-11-15T21:59:03,797 DEBUG [RS:1;ae3311ea5ccb:36535 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/ae3311ea5ccb:0, corePoolSize=1, maxPoolSize=1 2024-11-15T21:59:03,798 DEBUG [RS:1;ae3311ea5ccb:36535 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/ae3311ea5ccb:0, corePoolSize=1, maxPoolSize=1 2024-11-15T21:59:03,798 DEBUG [RS:1;ae3311ea5ccb:36535 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/ae3311ea5ccb:0, corePoolSize=1, maxPoolSize=1 2024-11-15T21:59:03,798 DEBUG [RS:1;ae3311ea5ccb:36535 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/ae3311ea5ccb:0, corePoolSize=1, maxPoolSize=1 2024-11-15T21:59:03,798 DEBUG [RS:1;ae3311ea5ccb:36535 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/ae3311ea5ccb:0, corePoolSize=2, maxPoolSize=2 2024-11-15T21:59:03,798 DEBUG [RS:1;ae3311ea5ccb:36535 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/ae3311ea5ccb:0, corePoolSize=1, maxPoolSize=1 2024-11-15T21:59:03,798 DEBUG [RS:1;ae3311ea5ccb:36535 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/ae3311ea5ccb:0, corePoolSize=1, maxPoolSize=1 2024-11-15T21:59:03,798 DEBUG [RS:1;ae3311ea5ccb:36535 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/ae3311ea5ccb:0, corePoolSize=1, maxPoolSize=1 2024-11-15T21:59:03,799 DEBUG [RS:1;ae3311ea5ccb:36535 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/ae3311ea5ccb:0, corePoolSize=1, maxPoolSize=1 2024-11-15T21:59:03,799 DEBUG [RS:1;ae3311ea5ccb:36535 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/ae3311ea5ccb:0, corePoolSize=1, maxPoolSize=1 2024-11-15T21:59:03,799 DEBUG [RS:1;ae3311ea5ccb:36535 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/ae3311ea5ccb:0, corePoolSize=1, maxPoolSize=1 2024-11-15T21:59:03,799 DEBUG [RS:1;ae3311ea5ccb:36535 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/ae3311ea5ccb:0, corePoolSize=3, maxPoolSize=3 2024-11-15T21:59:03,799 DEBUG [RS:1;ae3311ea5ccb:36535 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0, corePoolSize=3, maxPoolSize=3 2024-11-15T21:59:03,800 INFO [RS:1;ae3311ea5ccb:36535 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-15T21:59:03,801 INFO [RS:1;ae3311ea5ccb:36535 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-15T21:59:03,801 INFO [RS:1;ae3311ea5ccb:36535 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-15T21:59:03,801 INFO [RS:1;ae3311ea5ccb:36535 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-15T21:59:03,801 INFO [RS:1;ae3311ea5ccb:36535 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-15T21:59:03,801 INFO [RS:1;ae3311ea5ccb:36535 {}] hbase.ChoreService(168): Chore ScheduledChore name=ae3311ea5ccb,36535,1731707943240-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-15T21:59:03,810 INFO [RS:0;ae3311ea5ccb:41593 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-15T21:59:03,811 INFO [RS:0;ae3311ea5ccb:41593 {}] hbase.ChoreService(168): Chore ScheduledChore name=ae3311ea5ccb,41593,1731707943200-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-15T21:59:03,811 INFO [RS:0;ae3311ea5ccb:41593 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-15T21:59:03,811 INFO [RS:0;ae3311ea5ccb:41593 {}] regionserver.Replication(171): ae3311ea5ccb,41593,1731707943200 started 2024-11-15T21:59:03,820 INFO [RS:1;ae3311ea5ccb:36535 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-15T21:59:03,821 INFO [RS:1;ae3311ea5ccb:36535 {}] hbase.ChoreService(168): Chore ScheduledChore name=ae3311ea5ccb,36535,1731707943240-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-15T21:59:03,821 INFO [RS:1;ae3311ea5ccb:36535 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-15T21:59:03,821 INFO [RS:1;ae3311ea5ccb:36535 {}] regionserver.Replication(171): ae3311ea5ccb,36535,1731707943240 started 2024-11-15T21:59:03,829 INFO [RS:0;ae3311ea5ccb:41593 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-15T21:59:03,829 INFO [RS:0;ae3311ea5ccb:41593 {}] regionserver.HRegionServer(1482): Serving as ae3311ea5ccb,41593,1731707943200, RpcServer on ae3311ea5ccb/172.17.0.3:41593, sessionid=0x100f8692d830001 2024-11-15T21:59:03,829 DEBUG [RS:0;ae3311ea5ccb:41593 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-15T21:59:03,829 DEBUG [RS:0;ae3311ea5ccb:41593 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager ae3311ea5ccb,41593,1731707943200 2024-11-15T21:59:03,829 DEBUG [RS:0;ae3311ea5ccb:41593 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'ae3311ea5ccb,41593,1731707943200' 2024-11-15T21:59:03,829 DEBUG [RS:0;ae3311ea5ccb:41593 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-15T21:59:03,830 DEBUG [RS:0;ae3311ea5ccb:41593 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-15T21:59:03,831 DEBUG [RS:0;ae3311ea5ccb:41593 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-15T21:59:03,831 DEBUG [RS:0;ae3311ea5ccb:41593 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-15T21:59:03,831 DEBUG [RS:0;ae3311ea5ccb:41593 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager ae3311ea5ccb,41593,1731707943200 2024-11-15T21:59:03,831 DEBUG [RS:0;ae3311ea5ccb:41593 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'ae3311ea5ccb,41593,1731707943200' 2024-11-15T21:59:03,831 DEBUG [RS:0;ae3311ea5ccb:41593 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-15T21:59:03,831 DEBUG [RS:0;ae3311ea5ccb:41593 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-15T21:59:03,832 DEBUG [RS:0;ae3311ea5ccb:41593 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-15T21:59:03,832 INFO [RS:0;ae3311ea5ccb:41593 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-15T21:59:03,832 INFO [RS:0;ae3311ea5ccb:41593 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-15T21:59:03,839 INFO [RS:1;ae3311ea5ccb:36535 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-15T21:59:03,840 INFO [RS:1;ae3311ea5ccb:36535 {}] regionserver.HRegionServer(1482): Serving as ae3311ea5ccb,36535,1731707943240, RpcServer on ae3311ea5ccb/172.17.0.3:36535, sessionid=0x100f8692d830002 2024-11-15T21:59:03,840 DEBUG [RS:1;ae3311ea5ccb:36535 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-15T21:59:03,840 DEBUG [RS:1;ae3311ea5ccb:36535 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager ae3311ea5ccb,36535,1731707943240 2024-11-15T21:59:03,840 DEBUG [RS:1;ae3311ea5ccb:36535 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'ae3311ea5ccb,36535,1731707943240' 2024-11-15T21:59:03,840 DEBUG [RS:1;ae3311ea5ccb:36535 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-15T21:59:03,841 DEBUG [RS:1;ae3311ea5ccb:36535 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-15T21:59:03,842 DEBUG [RS:1;ae3311ea5ccb:36535 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-15T21:59:03,842 DEBUG [RS:1;ae3311ea5ccb:36535 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-15T21:59:03,842 DEBUG [RS:1;ae3311ea5ccb:36535 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager ae3311ea5ccb,36535,1731707943240 2024-11-15T21:59:03,842 DEBUG [RS:1;ae3311ea5ccb:36535 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'ae3311ea5ccb,36535,1731707943240' 2024-11-15T21:59:03,842 DEBUG [RS:1;ae3311ea5ccb:36535 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-15T21:59:03,842 DEBUG [RS:1;ae3311ea5ccb:36535 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-15T21:59:03,843 DEBUG [RS:1;ae3311ea5ccb:36535 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-15T21:59:03,843 INFO [RS:1;ae3311ea5ccb:36535 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-15T21:59:03,843 INFO [RS:1;ae3311ea5ccb:36535 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-15T21:59:03,864 WARN [ae3311ea5ccb:42653 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-15T21:59:03,933 INFO [RS:0;ae3311ea5ccb:41593 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-15T21:59:03,934 INFO [RS:0;ae3311ea5ccb:41593 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=ae3311ea5ccb%2C41593%2C1731707943200, suffix=, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3be1b200-2a9c-651b-3f6b-ac964062b7d6/f4b02001-5f5c-c384-44d5-c547c79a7f1b/WALs/ae3311ea5ccb,41593,1731707943200, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3be1b200-2a9c-651b-3f6b-ac964062b7d6/f4b02001-5f5c-c384-44d5-c547c79a7f1b/oldWALs, maxLogs=55 2024-11-15T21:59:03,944 INFO [RS:1;ae3311ea5ccb:36535 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-15T21:59:03,945 INFO [RS:1;ae3311ea5ccb:36535 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=ae3311ea5ccb%2C36535%2C1731707943240, suffix=, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3be1b200-2a9c-651b-3f6b-ac964062b7d6/f4b02001-5f5c-c384-44d5-c547c79a7f1b/WALs/ae3311ea5ccb,36535,1731707943240, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3be1b200-2a9c-651b-3f6b-ac964062b7d6/f4b02001-5f5c-c384-44d5-c547c79a7f1b/oldWALs, maxLogs=55 2024-11-15T21:59:03,956 INFO [RS:0;ae3311ea5ccb:41593 {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3be1b200-2a9c-651b-3f6b-ac964062b7d6/f4b02001-5f5c-c384-44d5-c547c79a7f1b/WALs/ae3311ea5ccb,41593,1731707943200/ae3311ea5ccb%2C41593%2C1731707943200.1731707943936 2024-11-15T21:59:03,956 DEBUG [RS:0;ae3311ea5ccb:41593 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-11-15T21:59:03,972 INFO [RS:1;ae3311ea5ccb:36535 {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3be1b200-2a9c-651b-3f6b-ac964062b7d6/f4b02001-5f5c-c384-44d5-c547c79a7f1b/WALs/ae3311ea5ccb,36535,1731707943240/ae3311ea5ccb%2C36535%2C1731707943240.1731707943947 2024-11-15T21:59:03,973 DEBUG [RS:1;ae3311ea5ccb:36535 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-11-15T21:59:04,115 DEBUG [ae3311ea5ccb:42653 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=2, allServersCount=2 2024-11-15T21:59:04,115 DEBUG [ae3311ea5ccb:42653 {}] balancer.BalancerClusterState(204): Hosts are {ae3311ea5ccb=0} racks are {/default-rack=0} 2024-11-15T21:59:04,116 DEBUG [ae3311ea5ccb:42653 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-11-15T21:59:04,116 DEBUG [ae3311ea5ccb:42653 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-11-15T21:59:04,116 DEBUG [ae3311ea5ccb:42653 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-11-15T21:59:04,116 DEBUG [ae3311ea5ccb:42653 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-11-15T21:59:04,116 INFO [ae3311ea5ccb:42653 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-11-15T21:59:04,116 INFO [ae3311ea5ccb:42653 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-11-15T21:59:04,116 DEBUG [ae3311ea5ccb:42653 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-11-15T21:59:04,117 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=ae3311ea5ccb,41593,1731707943200 2024-11-15T21:59:04,118 INFO [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as ae3311ea5ccb,41593,1731707943200, state=OPENING 2024-11-15T21:59:04,120 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2024-11-15T21:59:04,122 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42653-0x100f8692d830000, quorum=127.0.0.1:54544, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-15T21:59:04,122 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41593-0x100f8692d830001, quorum=127.0.0.1:54544, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-15T21:59:04,122 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36535-0x100f8692d830002, quorum=127.0.0.1:54544, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-15T21:59:04,123 DEBUG [PEWorker-3 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-15T21:59:04,123 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-15T21:59:04,123 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-15T21:59:04,123 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-15T21:59:04,123 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=ae3311ea5ccb,41593,1731707943200}] 2024-11-15T21:59:04,276 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-15T21:59:04,279 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-7-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:34149, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-15T21:59:04,282 INFO [RS_OPEN_META-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-11-15T21:59:04,282 INFO [RS_OPEN_META-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-15T21:59:04,282 INFO [RS_OPEN_META-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-11-15T21:59:04,283 INFO [RS_OPEN_META-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=ae3311ea5ccb%2C41593%2C1731707943200.meta, suffix=.meta, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3be1b200-2a9c-651b-3f6b-ac964062b7d6/f4b02001-5f5c-c384-44d5-c547c79a7f1b/WALs/ae3311ea5ccb,41593,1731707943200, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3be1b200-2a9c-651b-3f6b-ac964062b7d6/f4b02001-5f5c-c384-44d5-c547c79a7f1b/oldWALs, maxLogs=55 2024-11-15T21:59:04,305 INFO [RS_OPEN_META-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3be1b200-2a9c-651b-3f6b-ac964062b7d6/f4b02001-5f5c-c384-44d5-c547c79a7f1b/WALs/ae3311ea5ccb,41593,1731707943200/ae3311ea5ccb%2C41593%2C1731707943200.meta.1731707944284.meta 2024-11-15T21:59:04,307 DEBUG [RS_OPEN_META-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-11-15T21:59:04,307 DEBUG [RS_OPEN_META-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-11-15T21:59:04,307 DEBUG [RS_OPEN_META-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-11-15T21:59:04,307 DEBUG [RS_OPEN_META-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-11-15T21:59:04,308 INFO [RS_OPEN_META-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-11-15T21:59:04,308 DEBUG [RS_OPEN_META-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-11-15T21:59:04,308 DEBUG [RS_OPEN_META-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-15T21:59:04,308 DEBUG [RS_OPEN_META-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-11-15T21:59:04,308 DEBUG [RS_OPEN_META-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-11-15T21:59:04,309 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family info of region 1588230740 2024-11-15T21:59:04,309 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-15T21:59:04,310 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T21:59:04,310 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-15T21:59:04,310 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family ns of region 1588230740 2024-11-15T21:59:04,311 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-15T21:59:04,311 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T21:59:04,311 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-15T21:59:04,311 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family rep_barrier of region 1588230740 2024-11-15T21:59:04,311 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-15T21:59:04,311 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T21:59:04,312 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-15T21:59:04,312 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family table of region 1588230740 2024-11-15T21:59:04,312 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-15T21:59:04,312 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T21:59:04,312 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-15T21:59:04,313 DEBUG [RS_OPEN_META-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-15T21:59:04,313 DEBUG [RS_OPEN_META-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3be1b200-2a9c-651b-3f6b-ac964062b7d6/f4b02001-5f5c-c384-44d5-c547c79a7f1b/data/hbase/meta/1588230740 2024-11-15T21:59:04,314 DEBUG [RS_OPEN_META-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3be1b200-2a9c-651b-3f6b-ac964062b7d6/f4b02001-5f5c-c384-44d5-c547c79a7f1b/data/hbase/meta/1588230740 2024-11-15T21:59:04,314 DEBUG [RS_OPEN_META-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-15T21:59:04,314 DEBUG [RS_OPEN_META-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-15T21:59:04,314 DEBUG [RS_OPEN_META-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-15T21:59:04,315 DEBUG [RS_OPEN_META-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-15T21:59:04,315 INFO [RS_OPEN_META-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=71626574, jitterRate=0.0673191249370575}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-15T21:59:04,315 DEBUG [RS_OPEN_META-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-11-15T21:59:04,316 DEBUG [RS_OPEN_META-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1731707944308Writing region info on filesystem at 1731707944308Initializing all the Stores at 1731707944308Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731707944308Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731707944308Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731707944309 (+1 ms)Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731707944309Cleaning up temporary data from old regions at 1731707944314 (+5 ms)Running coprocessor post-open hooks at 1731707944315 (+1 ms)Region opened successfully at 1731707944316 (+1 ms) 2024-11-15T21:59:04,316 INFO [RS_OPEN_META-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1731707944276 2024-11-15T21:59:04,320 DEBUG [RS_OPEN_META-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-11-15T21:59:04,320 INFO [RS_OPEN_META-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-11-15T21:59:04,322 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=ae3311ea5ccb,41593,1731707943200 2024-11-15T21:59:04,323 INFO [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as ae3311ea5ccb,41593,1731707943200, state=OPEN 2024-11-15T21:59:04,325 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42653-0x100f8692d830000, quorum=127.0.0.1:54544, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-15T21:59:04,325 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-15T21:59:04,325 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=ae3311ea5ccb,41593,1731707943200 2024-11-15T21:59:04,325 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36535-0x100f8692d830002, quorum=127.0.0.1:54544, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-15T21:59:04,325 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-15T21:59:04,326 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41593-0x100f8692d830001, quorum=127.0.0.1:54544, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-15T21:59:04,326 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-15T21:59:04,328 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2 2024-11-15T21:59:04,328 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=ae3311ea5ccb,41593,1731707943200 in 202 msec 2024-11-15T21:59:04,330 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1 2024-11-15T21:59:04,330 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 617 msec 2024-11-15T21:59:04,331 DEBUG [PEWorker-2 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-15T21:59:04,331 INFO [PEWorker-2 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces 2024-11-15T21:59:04,338 DEBUG [PEWorker-2 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-15T21:59:04,338 DEBUG [PEWorker-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=ae3311ea5ccb,41593,1731707943200, seqNum=-1] 2024-11-15T21:59:04,338 DEBUG [PEWorker-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-15T21:59:04,340 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-7-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:41511, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-15T21:59:04,346 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 706 msec 2024-11-15T21:59:04,347 INFO [master/ae3311ea5ccb:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1731707944346, completionTime=-1 2024-11-15T21:59:04,347 INFO [master/ae3311ea5ccb:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=2; waited=0ms, expected min=2 server(s), max=2 server(s), master is running 2024-11-15T21:59:04,347 DEBUG [master/ae3311ea5ccb:0:becomeActiveMaster {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-15T21:59:04,349 INFO [master/ae3311ea5ccb:0:becomeActiveMaster {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-11-15T21:59:04,350 INFO [master/ae3311ea5ccb:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731708004350 2024-11-15T21:59:04,350 INFO [master/ae3311ea5ccb:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731708064350 2024-11-15T21:59:04,350 INFO [master/ae3311ea5ccb:0:becomeActiveMaster {}] assignment.AssignmentManager(1783): Joined the cluster in 2 msec 2024-11-15T21:59:04,350 INFO [master/ae3311ea5ccb:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ae3311ea5ccb,42653,1731707943146-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-15T21:59:04,350 INFO [master/ae3311ea5ccb:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ae3311ea5ccb,42653,1731707943146-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-15T21:59:04,351 INFO [master/ae3311ea5ccb:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ae3311ea5ccb,42653,1731707943146-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-15T21:59:04,351 INFO [master/ae3311ea5ccb:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-ae3311ea5ccb:42653, period=300000, unit=MILLISECONDS is enabled. 2024-11-15T21:59:04,351 INFO [master/ae3311ea5ccb:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2024-11-15T21:59:04,351 INFO [master/ae3311ea5ccb:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled. 2024-11-15T21:59:04,355 DEBUG [master/ae3311ea5ccb:0.Chore.1 {}] janitor.CatalogJanitor(180): 2024-11-15T21:59:04,356 INFO [master/ae3311ea5ccb:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 1.037sec 2024-11-15T21:59:04,357 INFO [master/ae3311ea5ccb:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2024-11-15T21:59:04,357 INFO [master/ae3311ea5ccb:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2024-11-15T21:59:04,357 INFO [master/ae3311ea5ccb:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2024-11-15T21:59:04,357 INFO [master/ae3311ea5ccb:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2024-11-15T21:59:04,357 INFO [master/ae3311ea5ccb:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2024-11-15T21:59:04,357 INFO [master/ae3311ea5ccb:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ae3311ea5ccb,42653,1731707943146-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-15T21:59:04,357 INFO [master/ae3311ea5ccb:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ae3311ea5ccb,42653,1731707943146-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2024-11-15T21:59:04,358 DEBUG [master/ae3311ea5ccb:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds 2024-11-15T21:59:04,358 INFO [master/ae3311ea5ccb:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2024-11-15T21:59:04,358 INFO [master/ae3311ea5ccb:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ae3311ea5ccb,42653,1731707943146-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-15T21:59:04,394 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@143f7fa, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T21:59:04,394 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request ae3311ea5ccb,42653,-1 for getting cluster id 2024-11-15T21:59:04,394 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-15T21:59:04,396 DEBUG [HMaster-EventLoopGroup-6-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '25790b20-82e5-471e-ad18-1eac11333487' 2024-11-15T21:59:04,397 DEBUG [RPCClient-NioEventLoopGroup-5-6 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-15T21:59:04,397 DEBUG [RPCClient-NioEventLoopGroup-5-6 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "25790b20-82e5-471e-ad18-1eac11333487" 2024-11-15T21:59:04,397 DEBUG [RPCClient-NioEventLoopGroup-5-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2699fa57, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T21:59:04,397 DEBUG [RPCClient-NioEventLoopGroup-5-6 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [ae3311ea5ccb,42653,-1] 2024-11-15T21:59:04,398 DEBUG [RPCClient-NioEventLoopGroup-5-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-15T21:59:04,398 DEBUG [RPCClient-NioEventLoopGroup-5-6 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T21:59:04,400 INFO [HMaster-EventLoopGroup-6-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:48206, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-15T21:59:04,401 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2af0bbc0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T21:59:04,402 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-15T21:59:04,403 DEBUG [RPCClient-NioEventLoopGroup-5-7 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=ae3311ea5ccb,41593,1731707943200, seqNum=-1] 2024-11-15T21:59:04,404 DEBUG [RPCClient-NioEventLoopGroup-5-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-15T21:59:04,406 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-7-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:50746, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-15T21:59:04,409 INFO [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=ae3311ea5ccb,42653,1731707943146 2024-11-15T21:59:04,409 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-15T21:59:04,410 DEBUG [RPCClient-NioEventLoopGroup-5-7 {}] client.AsyncConnectionImpl(321): The fetched master address is ae3311ea5ccb,42653,1731707943146 2024-11-15T21:59:04,410 DEBUG [RPCClient-NioEventLoopGroup-5-7 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@c4a0057 2024-11-15T21:59:04,410 DEBUG [RPCClient-NioEventLoopGroup-5-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-15T21:59:04,412 INFO [HMaster-EventLoopGroup-6-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:48220, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-15T21:59:04,413 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42653 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.3 create 'testBlockEvictionOnGracefulStop1731707944409', {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-15T21:59:04,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42653 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=testBlockEvictionOnGracefulStop1731707944409 2024-11-15T21:59:04,416 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=testBlockEvictionOnGracefulStop1731707944409 execute state=CREATE_TABLE_PRE_OPERATION 2024-11-15T21:59:04,416 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T21:59:04,417 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42653 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.3 procedure request for creating table: namespace: "default" qualifier: "testBlockEvictionOnGracefulStop1731707944409" procId is: 4 2024-11-15T21:59:04,417 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=testBlockEvictionOnGracefulStop1731707944409 execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-15T21:59:04,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42653 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-15T21:59:04,434 INFO [RegionOpenAndInit-testBlockEvictionOnGracefulStop1731707944409-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => d2cd4a927d1dc5c0d997e9855e15f923, NAME => 'testBlockEvictionOnGracefulStop1731707944409,,1731707944413.d2cd4a927d1dc5c0d997e9855e15f923.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testBlockEvictionOnGracefulStop1731707944409', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3be1b200-2a9c-651b-3f6b-ac964062b7d6/f4b02001-5f5c-c384-44d5-c547c79a7f1b 2024-11-15T21:59:04,441 DEBUG [RegionOpenAndInit-testBlockEvictionOnGracefulStop1731707944409-pool-0 {}] regionserver.HRegion(898): Instantiated testBlockEvictionOnGracefulStop1731707944409,,1731707944413.d2cd4a927d1dc5c0d997e9855e15f923.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-15T21:59:04,442 DEBUG [RegionOpenAndInit-testBlockEvictionOnGracefulStop1731707944409-pool-0 {}] regionserver.HRegion(1722): Closing d2cd4a927d1dc5c0d997e9855e15f923, disabling compactions & flushes 2024-11-15T21:59:04,442 INFO [RegionOpenAndInit-testBlockEvictionOnGracefulStop1731707944409-pool-0 {}] regionserver.HRegion(1755): Closing region testBlockEvictionOnGracefulStop1731707944409,,1731707944413.d2cd4a927d1dc5c0d997e9855e15f923. 2024-11-15T21:59:04,442 DEBUG [RegionOpenAndInit-testBlockEvictionOnGracefulStop1731707944409-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on testBlockEvictionOnGracefulStop1731707944409,,1731707944413.d2cd4a927d1dc5c0d997e9855e15f923. 2024-11-15T21:59:04,442 DEBUG [RegionOpenAndInit-testBlockEvictionOnGracefulStop1731707944409-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on testBlockEvictionOnGracefulStop1731707944409,,1731707944413.d2cd4a927d1dc5c0d997e9855e15f923. after waiting 0 ms 2024-11-15T21:59:04,442 DEBUG [RegionOpenAndInit-testBlockEvictionOnGracefulStop1731707944409-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region testBlockEvictionOnGracefulStop1731707944409,,1731707944413.d2cd4a927d1dc5c0d997e9855e15f923. 2024-11-15T21:59:04,442 INFO [RegionOpenAndInit-testBlockEvictionOnGracefulStop1731707944409-pool-0 {}] regionserver.HRegion(1973): Closed testBlockEvictionOnGracefulStop1731707944409,,1731707944413.d2cd4a927d1dc5c0d997e9855e15f923. 2024-11-15T21:59:04,442 DEBUG [RegionOpenAndInit-testBlockEvictionOnGracefulStop1731707944409-pool-0 {}] regionserver.HRegion(1676): Region close journal for d2cd4a927d1dc5c0d997e9855e15f923: Waiting for close lock at 1731707944442Disabling compacts and flushes for region at 1731707944442Disabling writes for close at 1731707944442Writing region close event to WAL at 1731707944442Closed at 1731707944442 2024-11-15T21:59:04,444 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=testBlockEvictionOnGracefulStop1731707944409 execute state=CREATE_TABLE_ADD_TO_META 2024-11-15T21:59:04,444 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testBlockEvictionOnGracefulStop1731707944409,,1731707944413.d2cd4a927d1dc5c0d997e9855e15f923.","families":{"info":[{"qualifier":"regioninfo","vlen":78,"tag":[],"timestamp":"1731707944444"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731707944444"}]},"ts":"1731707944444"} 2024-11-15T21:59:04,446 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-15T21:59:04,447 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=testBlockEvictionOnGracefulStop1731707944409 execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-15T21:59:04,448 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testBlockEvictionOnGracefulStop1731707944409","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731707944447"}]},"ts":"1731707944447"} 2024-11-15T21:59:04,450 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=testBlockEvictionOnGracefulStop1731707944409, state=ENABLING in hbase:meta 2024-11-15T21:59:04,451 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(204): Hosts are {ae3311ea5ccb=0} racks are {/default-rack=0} 2024-11-15T21:59:04,451 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-11-15T21:59:04,451 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-11-15T21:59:04,451 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-11-15T21:59:04,451 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-11-15T21:59:04,451 INFO [PEWorker-3 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-11-15T21:59:04,451 INFO [PEWorker-3 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-11-15T21:59:04,452 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-11-15T21:59:04,452 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1731707944409, region=d2cd4a927d1dc5c0d997e9855e15f923, ASSIGN}] 2024-11-15T21:59:04,454 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1731707944409, region=d2cd4a927d1dc5c0d997e9855e15f923, ASSIGN 2024-11-15T21:59:04,455 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1731707944409, region=d2cd4a927d1dc5c0d997e9855e15f923, ASSIGN; state=OFFLINE, location=ae3311ea5ccb,36535,1731707943240; forceNewPlan=false, retain=false 2024-11-15T21:59:04,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42653 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-15T21:59:04,605 INFO [ae3311ea5ccb:42653 {}] balancer.BaseLoadBalancer(388): Reassigned 1 regions. 1 retained the pre-restart assignment. 2024-11-15T21:59:04,606 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=d2cd4a927d1dc5c0d997e9855e15f923, regionState=OPENING, regionLocation=ae3311ea5ccb,36535,1731707943240 2024-11-15T21:59:04,610 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-8-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1731707944409, region=d2cd4a927d1dc5c0d997e9855e15f923, ASSIGN because future has completed 2024-11-15T21:59:04,611 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure d2cd4a927d1dc5c0d997e9855e15f923, server=ae3311ea5ccb,36535,1731707943240}] 2024-11-15T21:59:04,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42653 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-15T21:59:04,765 DEBUG [RSProcedureDispatcher-pool-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-15T21:59:04,768 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-8-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:42509, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-15T21:59:04,771 INFO [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(132): Open testBlockEvictionOnGracefulStop1731707944409,,1731707944413.d2cd4a927d1dc5c0d997e9855e15f923. 2024-11-15T21:59:04,771 DEBUG [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7752): Opening region: {ENCODED => d2cd4a927d1dc5c0d997e9855e15f923, NAME => 'testBlockEvictionOnGracefulStop1731707944409,,1731707944413.d2cd4a927d1dc5c0d997e9855e15f923.', STARTKEY => '', ENDKEY => ''} 2024-11-15T21:59:04,773 DEBUG [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testBlockEvictionOnGracefulStop1731707944409 d2cd4a927d1dc5c0d997e9855e15f923 2024-11-15T21:59:04,773 DEBUG [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(898): Instantiated testBlockEvictionOnGracefulStop1731707944409,,1731707944413.d2cd4a927d1dc5c0d997e9855e15f923.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-15T21:59:04,773 DEBUG [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7794): checking encryption for d2cd4a927d1dc5c0d997e9855e15f923 2024-11-15T21:59:04,773 DEBUG [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7797): checking classloading for d2cd4a927d1dc5c0d997e9855e15f923 2024-11-15T21:59:04,781 INFO [StoreOpener-d2cd4a927d1dc5c0d997e9855e15f923-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family family of region d2cd4a927d1dc5c0d997e9855e15f923 2024-11-15T21:59:04,791 INFO [StoreOpener-d2cd4a927d1dc5c0d997e9855e15f923-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region d2cd4a927d1dc5c0d997e9855e15f923 columnFamilyName family 2024-11-15T21:59:04,791 DEBUG [StoreOpener-d2cd4a927d1dc5c0d997e9855e15f923-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T21:59:04,791 INFO [StoreOpener-d2cd4a927d1dc5c0d997e9855e15f923-1 {}] regionserver.HStore(327): Store=d2cd4a927d1dc5c0d997e9855e15f923/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-15T21:59:04,791 DEBUG [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1038): replaying wal for d2cd4a927d1dc5c0d997e9855e15f923 2024-11-15T21:59:04,792 DEBUG [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3be1b200-2a9c-651b-3f6b-ac964062b7d6/f4b02001-5f5c-c384-44d5-c547c79a7f1b/data/default/testBlockEvictionOnGracefulStop1731707944409/d2cd4a927d1dc5c0d997e9855e15f923 2024-11-15T21:59:04,793 DEBUG [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3be1b200-2a9c-651b-3f6b-ac964062b7d6/f4b02001-5f5c-c384-44d5-c547c79a7f1b/data/default/testBlockEvictionOnGracefulStop1731707944409/d2cd4a927d1dc5c0d997e9855e15f923 2024-11-15T21:59:04,793 DEBUG [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1048): stopping wal replay for d2cd4a927d1dc5c0d997e9855e15f923 2024-11-15T21:59:04,793 DEBUG [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1060): Cleaning up temporary data for d2cd4a927d1dc5c0d997e9855e15f923 2024-11-15T21:59:04,794 DEBUG [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1093): writing seq id for d2cd4a927d1dc5c0d997e9855e15f923 2024-11-15T21:59:04,816 DEBUG [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3be1b200-2a9c-651b-3f6b-ac964062b7d6/f4b02001-5f5c-c384-44d5-c547c79a7f1b/data/default/testBlockEvictionOnGracefulStop1731707944409/d2cd4a927d1dc5c0d997e9855e15f923/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-15T21:59:04,817 INFO [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1114): Opened d2cd4a927d1dc5c0d997e9855e15f923; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=60407789, jitterRate=-0.09985379874706268}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-15T21:59:04,817 DEBUG [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1122): Running coprocessor post-open hooks for d2cd4a927d1dc5c0d997e9855e15f923 2024-11-15T21:59:04,819 DEBUG [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1006): Region open journal for d2cd4a927d1dc5c0d997e9855e15f923: Running coprocessor pre-open hook at 1731707944774Writing region info on filesystem at 1731707944774Initializing all the Stores at 1731707944774Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731707944774Cleaning up temporary data from old regions at 1731707944793 (+19 ms)Running coprocessor post-open hooks at 1731707944817 (+24 ms)Region opened successfully at 1731707944818 (+1 ms) 2024-11-15T21:59:04,820 INFO [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2236): Post open deploy tasks for testBlockEvictionOnGracefulStop1731707944409,,1731707944413.d2cd4a927d1dc5c0d997e9855e15f923., pid=6, masterSystemTime=1731707944764 2024-11-15T21:59:04,824 DEBUG [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2266): Finished post open deploy task for testBlockEvictionOnGracefulStop1731707944409,,1731707944413.d2cd4a927d1dc5c0d997e9855e15f923. 2024-11-15T21:59:04,824 INFO [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(153): Opened testBlockEvictionOnGracefulStop1731707944409,,1731707944413.d2cd4a927d1dc5c0d997e9855e15f923. 2024-11-15T21:59:04,826 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=d2cd4a927d1dc5c0d997e9855e15f923, regionState=OPEN, openSeqNum=2, regionLocation=ae3311ea5ccb,36535,1731707943240 2024-11-15T21:59:04,829 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-8-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure d2cd4a927d1dc5c0d997e9855e15f923, server=ae3311ea5ccb,36535,1731707943240 because future has completed 2024-11-15T21:59:04,837 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=6, resume processing ppid=5 2024-11-15T21:59:04,837 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure d2cd4a927d1dc5c0d997e9855e15f923, server=ae3311ea5ccb,36535,1731707943240 in 224 msec 2024-11-15T21:59:04,839 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=5, resume processing ppid=4 2024-11-15T21:59:04,839 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1731707944409, region=d2cd4a927d1dc5c0d997e9855e15f923, ASSIGN in 385 msec 2024-11-15T21:59:04,840 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=testBlockEvictionOnGracefulStop1731707944409 execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-15T21:59:04,841 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testBlockEvictionOnGracefulStop1731707944409","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731707944840"}]},"ts":"1731707944840"} 2024-11-15T21:59:04,844 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=testBlockEvictionOnGracefulStop1731707944409, state=ENABLED in hbase:meta 2024-11-15T21:59:04,845 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=testBlockEvictionOnGracefulStop1731707944409 execute state=CREATE_TABLE_POST_OPERATION 2024-11-15T21:59:04,846 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=testBlockEvictionOnGracefulStop1731707944409 in 431 msec 2024-11-15T21:59:05,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42653 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-15T21:59:05,048 INFO [RPCClient-NioEventLoopGroup-5-9 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:testBlockEvictionOnGracefulStop1731707944409 completed 2024-11-15T21:59:05,048 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table testBlockEvictionOnGracefulStop1731707944409 get assigned. Timeout = 60000ms 2024-11-15T21:59:05,048 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-15T21:59:05,053 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table testBlockEvictionOnGracefulStop1731707944409 assigned to meta. Checking AM states. 2024-11-15T21:59:05,053 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-15T21:59:05,054 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table testBlockEvictionOnGracefulStop1731707944409 assigned. 2024-11-15T21:59:05,059 DEBUG [RPCClient-NioEventLoopGroup-5-8 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testBlockEvictionOnGracefulStop1731707944409', row='row1', locateType=CURRENT is [region=testBlockEvictionOnGracefulStop1731707944409,,1731707944413.d2cd4a927d1dc5c0d997e9855e15f923., hostname=ae3311ea5ccb,36535,1731707943240, seqNum=2] 2024-11-15T21:59:05,060 DEBUG [RPCClient-NioEventLoopGroup-5-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-15T21:59:05,064 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-8-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:56408, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-15T21:59:05,078 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing d2cd4a927d1dc5c0d997e9855e15f923 1/1 column families, dataSize=78 B heapSize=480 B 2024-11-15T21:59:05,102 DEBUG [Time-limited test {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on ebf6a2ec462e4af99422abc19ec14176 2024-11-15T21:59:05,118 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3be1b200-2a9c-651b-3f6b-ac964062b7d6/f4b02001-5f5c-c384-44d5-c547c79a7f1b/data/default/testBlockEvictionOnGracefulStop1731707944409/d2cd4a927d1dc5c0d997e9855e15f923/.tmp/family/ebf6a2ec462e4af99422abc19ec14176 is 43, key is row1/family:qf1/1/Put/seqid=0 2024-11-15T21:59:05,119 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=78 B at sequenceid=6 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3be1b200-2a9c-651b-3f6b-ac964062b7d6/f4b02001-5f5c-c384-44d5-c547c79a7f1b/data/default/testBlockEvictionOnGracefulStop1731707944409/d2cd4a927d1dc5c0d997e9855e15f923/.tmp/family/ebf6a2ec462e4af99422abc19ec14176 2024-11-15T21:59:05,122 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3be1b200-2a9c-651b-3f6b-ac964062b7d6/f4b02001-5f5c-c384-44d5-c547c79a7f1b/data/default/testBlockEvictionOnGracefulStop1731707944409/d2cd4a927d1dc5c0d997e9855e15f923/.tmp/family/ebf6a2ec462e4af99422abc19ec14176 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3be1b200-2a9c-651b-3f6b-ac964062b7d6/f4b02001-5f5c-c384-44d5-c547c79a7f1b/data/default/testBlockEvictionOnGracefulStop1731707944409/d2cd4a927d1dc5c0d997e9855e15f923/family/ebf6a2ec462e4af99422abc19ec14176 2024-11-15T21:59:05,124 DEBUG [Time-limited test {}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3be1b200-2a9c-651b-3f6b-ac964062b7d6/f4b02001-5f5c-c384-44d5-c547c79a7f1b/data/default/testBlockEvictionOnGracefulStop1731707944409/d2cd4a927d1dc5c0d997e9855e15f923/family/ebf6a2ec462e4af99422abc19ec14176, delay=943 ms 2024-11-15T21:59:05,125 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3be1b200-2a9c-651b-3f6b-ac964062b7d6/f4b02001-5f5c-c384-44d5-c547c79a7f1b/data/default/testBlockEvictionOnGracefulStop1731707944409/d2cd4a927d1dc5c0d997e9855e15f923/family/ebf6a2ec462e4af99422abc19ec14176, entries=2, sequenceid=6, filesize=4.9 K 2024-11-15T21:59:05,126 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~78 B/78, heapSize ~464 B/464, currentSize=0 B/0 for d2cd4a927d1dc5c0d997e9855e15f923 in 48ms, sequenceid=6, compaction requested=false 2024-11-15T21:59:05,126 DEBUG [Time-limited test {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'testBlockEvictionOnGracefulStop1731707944409' 2024-11-15T21:59:05,127 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for d2cd4a927d1dc5c0d997e9855e15f923: 2024-11-15T21:59:05,340 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-15T21:59:06,068 DEBUG [hfile-prefetch-1731707942558 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key ebf6a2ec462e4af99422abc19ec14176_0. 2024-11-15T21:59:06,068 DEBUG [hfile-prefetch-1731707942558 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key ebf6a2ec462e4af99422abc19ec14176_117. 2024-11-15T21:59:06,069 DEBUG [hfile-prefetch-1731707942558 {}] bucket.BucketCache(2281): Notifying caching completed for file file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3be1b200-2a9c-651b-3f6b-ac964062b7d6/f4b02001-5f5c-c384-44d5-c547c79a7f1b/data/default/testBlockEvictionOnGracefulStop1731707944409/d2cd4a927d1dc5c0d997e9855e15f923/family/ebf6a2ec462e4af99422abc19ec14176, with total blocks 2, and data blocks 1 2024-11-15T21:59:06,069 DEBUG [hfile-prefetch-1731707942558 {}] bucket.BucketCache(2285): iterating over 0 entries in the backing map 2024-11-15T21:59:06,069 DEBUG [hfile-prefetch-1731707942558 {}] bucket.BucketCache(2308): Prefetch executor completed for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3be1b200-2a9c-651b-3f6b-ac964062b7d6/f4b02001-5f5c-c384-44d5-c547c79a7f1b/data/default/testBlockEvictionOnGracefulStop1731707944409/d2cd4a927d1dc5c0d997e9855e15f923/family/ebf6a2ec462e4af99422abc19ec14176, but only 0 data blocks were cached. Total data blocks for file: 1. Checking for blocks pending cache in cache writer queue. 2024-11-15T21:59:06,069 DEBUG [hfile-prefetch-1731707942558 {}] bucket.BucketCache(2318): There are still blocks pending caching for file file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3be1b200-2a9c-651b-3f6b-ac964062b7d6/f4b02001-5f5c-c384-44d5-c547c79a7f1b/data/default/testBlockEvictionOnGracefulStop1731707944409/d2cd4a927d1dc5c0d997e9855e15f923/family/ebf6a2ec462e4af99422abc19ec14176. Will sleep 100ms and try the verification again. 2024-11-15T21:59:06,114 DEBUG [bucket-cache-persister {}] bucket.BucketCachePersister(47): Cache is inconsistent, persisting to disk 2024-11-15T21:59:06,114 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1487): Thread bucket-cache-persister started persisting bucket cache to file 2024-11-15T21:59:06,114 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1494): Persist in new chunked persistence format. 2024-11-15T21:59:06,114 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1673): persistToFile: before persisting backing map size: 0, fullycachedFiles size: 0, chunkSize: 10000 2024-11-15T21:59:06,117 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1680): persistToFile: after persisting backing map size: 0, fullycachedFiles size: 0 2024-11-15T21:59:06,117 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1498): PersistToFile: after persisting backing map size: 0, fullycachedFiles size: 0, file name: bucket.persistence1731707946114 2024-11-15T21:59:06,117 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1509): Thread bucket-cache-persister finished persisting bucket cache to file, renaming 2024-11-15T21:59:06,128 INFO [Time-limited test {}] hbase.SingleProcessHBaseCluster(504): Stopping Thread[RS:1;ae3311ea5ccb:36535,5,FailOnTimeoutGroup] 2024-11-15T21:59:06,128 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server 'ae3311ea5ccb,36535,1731707943240' ***** 2024-11-15T21:59:06,128 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Stopping rs 1 2024-11-15T21:59:06,128 INFO [RS:1;ae3311ea5ccb:36535 {}] bucket.BucketCache(1738): Disabling cache 2024-11-15T21:59:06,129 INFO [RS:1;ae3311ea5ccb:36535 {}] bucket.BucketCache(1762): Shutdown bucket cache: IO persistent=true; path to write=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f706088d-a2ef-5fec-6a8f-fdcb2a34ea2f/bucket.persistence 2024-11-15T21:59:06,129 INFO [Time-limited test-BucketCacheWriter-1 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-1 exiting, cacheEnabled=false 2024-11-15T21:59:06,129 INFO [Time-limited test-BucketCacheWriter-0 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-0 exiting, cacheEnabled=false 2024-11-15T21:59:06,129 INFO [Time-limited test-BucketCacheWriter-2 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-2 exiting, cacheEnabled=false 2024-11-15T21:59:06,129 INFO [RS:1;ae3311ea5ccb:36535 {}] bucket.BucketCache(1768): Shutting down cache persister thread. 2024-11-15T21:59:06,129 WARN [bucket-cache-persister {}] bucket.BucketCachePersister(64): Interrupting BucketCachePersister thread. java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCachePersister.run(BucketCachePersister.java:45) ~[classes/:?] 2024-11-15T21:59:06,139 DEBUG [RS:1;ae3311ea5ccb:36535 {}] bucket.BucketCache(1487): Thread RS:1;ae3311ea5ccb:36535 started persisting bucket cache to file 2024-11-15T21:59:06,139 DEBUG [RS:1;ae3311ea5ccb:36535 {}] bucket.BucketCache(1494): Persist in new chunked persistence format. 2024-11-15T21:59:06,139 DEBUG [RS:1;ae3311ea5ccb:36535 {}] bucket.BucketCache(1673): persistToFile: before persisting backing map size: 1, fullycachedFiles size: 0, chunkSize: 10000 2024-11-15T21:59:06,142 DEBUG [RS:1;ae3311ea5ccb:36535 {}] bucket.BucketCache(1680): persistToFile: after persisting backing map size: 1, fullycachedFiles size: 0 2024-11-15T21:59:06,143 DEBUG [RS:1;ae3311ea5ccb:36535 {}] bucket.BucketCache(1498): PersistToFile: after persisting backing map size: 1, fullycachedFiles size: 0, file name: bucket.persistence1731707946139 2024-11-15T21:59:06,143 DEBUG [RS:1;ae3311ea5ccb:36535 {}] bucket.BucketCache(1509): Thread RS:1;ae3311ea5ccb:36535 finished persisting bucket cache to file, renaming 2024-11-15T21:59:06,143 INFO [RS:1;ae3311ea5ccb:36535 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-15T21:59:06,143 INFO [RS:1;ae3311ea5ccb:36535 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-15T21:59:06,143 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-15T21:59:06,143 INFO [RS:1;ae3311ea5ccb:36535 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-15T21:59:06,143 INFO [RS:1;ae3311ea5ccb:36535 {}] regionserver.HRegionServer(3091): Received CLOSE for d2cd4a927d1dc5c0d997e9855e15f923 2024-11-15T21:59:06,143 INFO [RS:1;ae3311ea5ccb:36535 {}] regionserver.HRegionServer(959): stopping server ae3311ea5ccb,36535,1731707943240 2024-11-15T21:59:06,143 INFO [RS:1;ae3311ea5ccb:36535 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-15T21:59:06,143 INFO [RS:1;ae3311ea5ccb:36535 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:1;ae3311ea5ccb:36535. 2024-11-15T21:59:06,143 INFO [RS_CLOSE_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1662): Closing region testBlockEvictionOnGracefulStop1731707944409,,1731707944413.d2cd4a927d1dc5c0d997e9855e15f923. during a graceful stop, and cache persistence is on, so setting evict on close to false. 2024-11-15T21:59:06,144 DEBUG [RS_CLOSE_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing d2cd4a927d1dc5c0d997e9855e15f923, disabling compactions & flushes 2024-11-15T21:59:06,144 DEBUG [RS:1;ae3311ea5ccb:36535 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-15T21:59:06,144 INFO [RS_CLOSE_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testBlockEvictionOnGracefulStop1731707944409,,1731707944413.d2cd4a927d1dc5c0d997e9855e15f923. 2024-11-15T21:59:06,144 DEBUG [RS_CLOSE_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testBlockEvictionOnGracefulStop1731707944409,,1731707944413.d2cd4a927d1dc5c0d997e9855e15f923. 2024-11-15T21:59:06,144 DEBUG [RS:1;ae3311ea5ccb:36535 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T21:59:06,144 DEBUG [RS_CLOSE_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testBlockEvictionOnGracefulStop1731707944409,,1731707944413.d2cd4a927d1dc5c0d997e9855e15f923. after waiting 0 ms 2024-11-15T21:59:06,144 DEBUG [RS_CLOSE_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testBlockEvictionOnGracefulStop1731707944409,,1731707944413.d2cd4a927d1dc5c0d997e9855e15f923. 2024-11-15T21:59:06,144 INFO [RS:1;ae3311ea5ccb:36535 {}] regionserver.HRegionServer(1321): Waiting on 1 regions to close 2024-11-15T21:59:06,144 DEBUG [RS:1;ae3311ea5ccb:36535 {}] regionserver.HRegionServer(1325): Online Regions={d2cd4a927d1dc5c0d997e9855e15f923=testBlockEvictionOnGracefulStop1731707944409,,1731707944413.d2cd4a927d1dc5c0d997e9855e15f923.} 2024-11-15T21:59:06,144 DEBUG [RS:1;ae3311ea5ccb:36535 {}] regionserver.HRegionServer(1351): Waiting on d2cd4a927d1dc5c0d997e9855e15f923 2024-11-15T21:59:06,145 DEBUG [StoreFileCloser-d2cd4a927d1dc5c0d997e9855e15f923-family-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3be1b200-2a9c-651b-3f6b-ac964062b7d6/f4b02001-5f5c-c384-44d5-c547c79a7f1b/data/default/testBlockEvictionOnGracefulStop1731707944409/d2cd4a927d1dc5c0d997e9855e15f923/family/ebf6a2ec462e4af99422abc19ec14176 2024-11-15T21:59:06,145 WARN [hfile-prefetch-1731707942558 {}] hfile.HFilePreadReader$1(158): Prefetch path=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3be1b200-2a9c-651b-3f6b-ac964062b7d6/f4b02001-5f5c-c384-44d5-c547c79a7f1b/data/default/testBlockEvictionOnGracefulStop1731707944409/d2cd4a927d1dc5c0d997e9855e15f923/family/ebf6a2ec462e4af99422abc19ec14176, offset=158, end=158 java.lang.RuntimeException: java.lang.InterruptedException: sleep interrupted at org.apache.hadoop.hbase.io.hfile.bucket.BucketCache.notifyFileCachingCompleted(BucketCache.java:2337) ~[classes/:?] at org.apache.hadoop.hbase.io.hfile.CombinedBlockCache.notifyFileCachingCompleted(CombinedBlockCache.java:469) ~[classes/:?] at org.apache.hadoop.hbase.io.hfile.HFilePreadReader$1.run(HFilePreadReader.java:148) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.lambda$tracedRunnable$2(TraceUtil.java:155) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:539) ~[?:?] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:304) ~[?:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCache.notifyFileCachingCompleted(BucketCache.java:2320) ~[classes/:?] ... 9 more 2024-11-15T21:59:06,145 DEBUG [hfile-prefetch-1731707942558 {}] hfile.PrefetchExecutor(121): Prefetch completed for ebf6a2ec462e4af99422abc19ec14176 2024-11-15T21:59:06,149 DEBUG [RS_CLOSE_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3be1b200-2a9c-651b-3f6b-ac964062b7d6/f4b02001-5f5c-c384-44d5-c547c79a7f1b/data/default/testBlockEvictionOnGracefulStop1731707944409/d2cd4a927d1dc5c0d997e9855e15f923/recovered.edits/9.seqid, newMaxSeqId=9, maxSeqId=1 2024-11-15T21:59:06,150 INFO [RS_CLOSE_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testBlockEvictionOnGracefulStop1731707944409,,1731707944413.d2cd4a927d1dc5c0d997e9855e15f923. 2024-11-15T21:59:06,150 DEBUG [RS_CLOSE_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for d2cd4a927d1dc5c0d997e9855e15f923: Waiting for close lock at 1731707946143Running coprocessor pre-close hooks at 1731707946144 (+1 ms)Disabling compacts and flushes for region at 1731707946144Disabling writes for close at 1731707946144Writing region close event to WAL at 1731707946145 (+1 ms)Running coprocessor post-close hooks at 1731707946150 (+5 ms)Closed at 1731707946150 2024-11-15T21:59:06,150 DEBUG [RS_CLOSE_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testBlockEvictionOnGracefulStop1731707944409,,1731707944413.d2cd4a927d1dc5c0d997e9855e15f923. 2024-11-15T21:59:06,204 INFO [regionserver/ae3311ea5ccb:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-15T21:59:06,344 INFO [RS:1;ae3311ea5ccb:36535 {}] regionserver.HRegionServer(976): stopping server ae3311ea5ccb,36535,1731707943240; all regions closed. 2024-11-15T21:59:06,346 DEBUG [RS:1;ae3311ea5ccb:36535 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3be1b200-2a9c-651b-3f6b-ac964062b7d6/f4b02001-5f5c-c384-44d5-c547c79a7f1b/oldWALs 2024-11-15T21:59:06,346 INFO [RS:1;ae3311ea5ccb:36535 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL ae3311ea5ccb%2C36535%2C1731707943240:(num 1731707943947) 2024-11-15T21:59:06,346 DEBUG [RS:1;ae3311ea5ccb:36535 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T21:59:06,346 INFO [RS:1;ae3311ea5ccb:36535 {}] regionserver.LeaseManager(133): Closed leases 2024-11-15T21:59:06,346 INFO [RS:1;ae3311ea5ccb:36535 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-15T21:59:06,347 INFO [RS:1;ae3311ea5ccb:36535 {}] hbase.ChoreService(370): Chore service for: regionserver/ae3311ea5ccb:0 had [ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS] on shutdown 2024-11-15T21:59:06,347 INFO [RS:1;ae3311ea5ccb:36535 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-15T21:59:06,347 INFO [RS:1;ae3311ea5ccb:36535 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-15T21:59:06,347 INFO [RS:1;ae3311ea5ccb:36535 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-15T21:59:06,347 INFO [regionserver/ae3311ea5ccb:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-15T21:59:06,347 INFO [RS:1;ae3311ea5ccb:36535 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-15T21:59:06,347 INFO [RS:1;ae3311ea5ccb:36535 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.3:36535 2024-11-15T21:59:06,365 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42653-0x100f8692d830000, quorum=127.0.0.1:54544, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-15T21:59:06,365 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36535-0x100f8692d830002, quorum=127.0.0.1:54544, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/ae3311ea5ccb,36535,1731707943240 2024-11-15T21:59:06,365 INFO [RS:1;ae3311ea5ccb:36535 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-15T21:59:06,380 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [ae3311ea5ccb,36535,1731707943240] 2024-11-15T21:59:06,396 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/ae3311ea5ccb,36535,1731707943240 already deleted, retry=false 2024-11-15T21:59:06,396 INFO [RegionServerTracker-0 {}] master.ServerManager(695): Processing expiration of ae3311ea5ccb,36535,1731707943240 on ae3311ea5ccb,42653,1731707943146 2024-11-15T21:59:06,401 DEBUG [RegionServerTracker-0 {}] procedure2.ProcedureExecutor(1139): Stored pid=7, state=RUNNABLE:SERVER_CRASH_START, hasLock=false; ServerCrashProcedure ae3311ea5ccb,36535,1731707943240, splitWal=true, meta=false 2024-11-15T21:59:06,403 INFO [PEWorker-1 {}] procedure.ServerCrashProcedure(169): Start pid=7, state=RUNNABLE:SERVER_CRASH_START, hasLock=true; ServerCrashProcedure ae3311ea5ccb,36535,1731707943240, splitWal=true, meta=false 2024-11-15T21:59:06,403 INFO [RegionServerTracker-0 {}] assignment.AssignmentManager(1999): Scheduled ServerCrashProcedure pid=7 for ae3311ea5ccb,36535,1731707943240 (carryingMeta=false) ae3311ea5ccb,36535,1731707943240/CRASHED/regionCount=1/lock=java.util.concurrent.locks.ReentrantReadWriteLock@61b744ed[Write locks = 1, Read locks = 0], oldState=ONLINE. 2024-11-15T21:59:06,404 INFO [PEWorker-1 {}] procedure.ServerCrashProcedure(207): ae3311ea5ccb,36535,1731707943240 had 1 regions 2024-11-15T21:59:06,405 INFO [PEWorker-1 {}] procedure.ServerCrashProcedure(339): Splitting WALs pid=7, state=RUNNABLE:SERVER_CRASH_SPLIT_LOGS, hasLock=true; ServerCrashProcedure ae3311ea5ccb,36535,1731707943240, splitWal=true, meta=false, isMeta: false 2024-11-15T21:59:06,405 DEBUG [PEWorker-1 {}] master.MasterWalManager(329): Renamed region directory: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3be1b200-2a9c-651b-3f6b-ac964062b7d6/f4b02001-5f5c-c384-44d5-c547c79a7f1b/WALs/ae3311ea5ccb,36535,1731707943240-splitting 2024-11-15T21:59:06,406 INFO [PEWorker-1 {}] master.SplitLogManager(171): file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3be1b200-2a9c-651b-3f6b-ac964062b7d6/f4b02001-5f5c-c384-44d5-c547c79a7f1b/WALs/ae3311ea5ccb,36535,1731707943240-splitting dir is empty, no logs to split. 2024-11-15T21:59:06,406 INFO [PEWorker-1 {}] master.SplitWALManager(105): ae3311ea5ccb,36535,1731707943240 WAL count=0, meta=false 2024-11-15T21:59:06,407 INFO [PEWorker-1 {}] master.SplitLogManager(171): file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3be1b200-2a9c-651b-3f6b-ac964062b7d6/f4b02001-5f5c-c384-44d5-c547c79a7f1b/WALs/ae3311ea5ccb,36535,1731707943240-splitting dir is empty, no logs to split. 2024-11-15T21:59:06,407 INFO [PEWorker-1 {}] master.SplitWALManager(105): ae3311ea5ccb,36535,1731707943240 WAL count=0, meta=false 2024-11-15T21:59:06,407 DEBUG [PEWorker-1 {}] procedure.ServerCrashProcedure(329): Check if ae3311ea5ccb,36535,1731707943240 WAL splitting is done? wals=0, meta=false 2024-11-15T21:59:06,408 INFO [PEWorker-1 {}] procedure.ServerCrashProcedure(321): Remove WAL directory for ae3311ea5ccb,36535,1731707943240 failed, ignore...File file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3be1b200-2a9c-651b-3f6b-ac964062b7d6/f4b02001-5f5c-c384-44d5-c547c79a7f1b/WALs/ae3311ea5ccb,36535,1731707943240-splitting does not exist 2024-11-15T21:59:06,410 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=8, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1731707944409, region=d2cd4a927d1dc5c0d997e9855e15f923, ASSIGN}] 2024-11-15T21:59:06,410 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=8, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1731707944409, region=d2cd4a927d1dc5c0d997e9855e15f923, ASSIGN 2024-11-15T21:59:06,411 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(269): Starting pid=8, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1731707944409, region=d2cd4a927d1dc5c0d997e9855e15f923, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-11-15T21:59:06,480 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36535-0x100f8692d830002, quorum=127.0.0.1:54544, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-15T21:59:06,480 INFO [RS:1;ae3311ea5ccb:36535 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-15T21:59:06,480 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36535-0x100f8692d830002, quorum=127.0.0.1:54544, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-15T21:59:06,480 INFO [RS:1;ae3311ea5ccb:36535 {}] regionserver.HRegionServer(1031): Exiting; stopping=ae3311ea5ccb,36535,1731707943240; zookeeper connection closed. 2024-11-15T21:59:06,481 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@2349bb53 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@2349bb53 2024-11-15T21:59:06,562 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=8 updating hbase:meta row=d2cd4a927d1dc5c0d997e9855e15f923, regionState=OPENING, regionLocation=ae3311ea5ccb,41593,1731707943200 2024-11-15T21:59:06,564 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-8-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=8, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1731707944409, region=d2cd4a927d1dc5c0d997e9855e15f923, ASSIGN because future has completed 2024-11-15T21:59:06,565 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=9, ppid=8, state=RUNNABLE, hasLock=false; OpenRegionProcedure d2cd4a927d1dc5c0d997e9855e15f923, server=ae3311ea5ccb,41593,1731707943200}] 2024-11-15T21:59:06,655 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/ae3311ea5ccb:0 server-side Connection retries=45 2024-11-15T21:59:06,655 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-15T21:59:06,655 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-15T21:59:06,655 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-15T21:59:06,655 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-15T21:59:06,655 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-15T21:59:06,655 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-15T21:59:06,656 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-15T21:59:06,656 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.3:45793 2024-11-15T21:59:06,659 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:45793 connecting to ZooKeeper ensemble=127.0.0.1:54544 2024-11-15T21:59:06,659 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-15T21:59:06,659 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-15T21:59:06,666 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:457930x0, quorum=127.0.0.1:54544, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-15T21:59:06,666 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:45793-0x100f8692d830003 connected 2024-11-15T21:59:06,666 DEBUG [pool-297-thread-1 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: INIT 2024-11-15T21:59:06,670 DEBUG [Time-limited test {}] zookeeper.ZKUtil(111): regionserver:45793-0x100f8692d830003, quorum=127.0.0.1:54544, baseZNode=/hbase Set watcher on existing znode=/hbase/running 2024-11-15T21:59:06,671 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-15T21:59:06,671 INFO [Time-limited test {}] bucket.FileIOEngine(99): Allocating cache 400 MB, on the path:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f706088d-a2ef-5fec-6a8f-fdcb2a34ea2f/bucket.cache 2024-11-15T21:59:06,671 INFO [Time-limited test {}] bucket.BucketCache(364): Instantiating BucketCache with acceptableFactor: 0.95, minFactor: 0.85, extraFreeFactor: 0.1, singleFactor: 0.25, multiFactor: 0.5, memoryFactor: 0.25, useStrongRef: false 2024-11-15T21:59:06,672 INFO [Time-limited test {}] bucket.BucketCache(478): Starting BucketCachePersisterThread 2024-11-15T21:59:06,672 INFO [Time-limited test {}] bucket.BucketCachePersister(38): BucketCachePersister started with interval: 100 2024-11-15T21:59:06,672 INFO [Thread-201 {}] bucket.BucketCache(1530): Started retrieving bucket cache from file 2024-11-15T21:59:06,673 INFO [Thread-201 {}] bucket.BucketCache(1552): Reading new chunked format of persistence. 2024-11-15T21:59:06,673 INFO [Time-limited test {}] bucket.BucketCache(403): Started bucket cache; ioengine=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f706088d-a2ef-5fec-6a8f-fdcb2a34ea2f/bucket.cache, capacity=400 MB, blockSize=64 KB, writerThreadNum=3, writerQLen=64, persistencePath=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f706088d-a2ef-5fec-6a8f-fdcb2a34ea2f/bucket.persistence, bucketAllocator=org.apache.hadoop.hbase.io.hfile.bucket.BucketAllocator 2024-11-15T21:59:06,674 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-15T21:59:06,674 INFO [Thread-201 {}] bucket.BucketCache(1705): Retrieved 1 of chunks with blockCount = 1. 2024-11-15T21:59:06,675 DEBUG [Time-limited test {}] zookeeper.ZKUtil(111): regionserver:45793-0x100f8692d830003, quorum=127.0.0.1:54544, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-11-15T21:59:06,677 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:45793-0x100f8692d830003, quorum=127.0.0.1:54544, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-15T21:59:06,677 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=45793 2024-11-15T21:59:06,678 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=45793 2024-11-15T21:59:06,678 DEBUG [Thread-201 {}] bucket.BucketCache(1569): Updating region size map after retrieving cached file list 2024-11-15T21:59:06,678 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=45793 2024-11-15T21:59:06,678 INFO [Thread-201 {}] bucket.BucketAllocator(343): Cache totalSize=418148352, buckets=199, bucket capacity=2101248=(4*525312)=(FEWEST_ITEMS_IN_BUCKET*(largest configured bucketcache size)) 2024-11-15T21:59:06,678 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=45793 2024-11-15T21:59:06,678 INFO [Thread-201 {}] bucket.BucketCache(1562): Bucket cache retrieved from file successfully with size: 1 2024-11-15T21:59:06,678 INFO [Thread-201 {}] bucket.BucketCache(413): Persistent bucket cache recovery from /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f706088d-a2ef-5fec-6a8f-fdcb2a34ea2f/bucket.persistence is complete. 2024-11-15T21:59:06,679 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=45793 2024-11-15T21:59:06,680 INFO [RS:2;ae3311ea5ccb:45793 {}] regionserver.HRegionServer(746): ClusterId : 25790b20-82e5-471e-ad18-1eac11333487 2024-11-15T21:59:06,680 DEBUG [RS:2;ae3311ea5ccb:45793 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-15T21:59:06,683 DEBUG [RS:2;ae3311ea5ccb:45793 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-15T21:59:06,683 DEBUG [RS:2;ae3311ea5ccb:45793 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-15T21:59:06,685 DEBUG [RS:2;ae3311ea5ccb:45793 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-15T21:59:06,685 DEBUG [RS:2;ae3311ea5ccb:45793 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@270af354, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=ae3311ea5ccb/172.17.0.3:0 2024-11-15T21:59:06,698 DEBUG [RS:2;ae3311ea5ccb:45793 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:2;ae3311ea5ccb:45793 2024-11-15T21:59:06,698 INFO [RS:2;ae3311ea5ccb:45793 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-15T21:59:06,698 INFO [RS:2;ae3311ea5ccb:45793 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-15T21:59:06,698 DEBUG [RS:2;ae3311ea5ccb:45793 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-15T21:59:06,699 INFO [RS:2;ae3311ea5ccb:45793 {}] regionserver.HRegionServer(2659): reportForDuty to master=ae3311ea5ccb,42653,1731707943146 with port=45793, startcode=1731707946654 2024-11-15T21:59:06,699 DEBUG [RS:2;ae3311ea5ccb:45793 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-15T21:59:06,701 INFO [HMaster-EventLoopGroup-6-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:55843, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=RegionServerStatusService 2024-11-15T21:59:06,702 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=42653 {}] master.ServerManager(363): Checking decommissioned status of RegionServer ae3311ea5ccb,45793,1731707946654 2024-11-15T21:59:06,702 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=42653 {}] master.ServerManager(517): Registering regionserver=ae3311ea5ccb,45793,1731707946654 2024-11-15T21:59:06,703 DEBUG [RS:2;ae3311ea5ccb:45793 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3be1b200-2a9c-651b-3f6b-ac964062b7d6/f4b02001-5f5c-c384-44d5-c547c79a7f1b 2024-11-15T21:59:06,703 DEBUG [RS:2;ae3311ea5ccb:45793 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=file:/ 2024-11-15T21:59:06,703 DEBUG [RS:2;ae3311ea5ccb:45793 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-15T21:59:06,705 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42653-0x100f8692d830000, quorum=127.0.0.1:54544, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-15T21:59:06,706 DEBUG [RS:2;ae3311ea5ccb:45793 {}] zookeeper.ZKUtil(111): regionserver:45793-0x100f8692d830003, quorum=127.0.0.1:54544, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/ae3311ea5ccb,45793,1731707946654 2024-11-15T21:59:06,706 WARN [RS:2;ae3311ea5ccb:45793 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-15T21:59:06,706 INFO [RS:2;ae3311ea5ccb:45793 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-15T21:59:06,706 DEBUG [RS:2;ae3311ea5ccb:45793 {}] regionserver.HRegionServer(1793): logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3be1b200-2a9c-651b-3f6b-ac964062b7d6/f4b02001-5f5c-c384-44d5-c547c79a7f1b/WALs/ae3311ea5ccb,45793,1731707946654 2024-11-15T21:59:06,706 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [ae3311ea5ccb,45793,1731707946654] 2024-11-15T21:59:06,713 INFO [RS:2;ae3311ea5ccb:45793 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-15T21:59:06,717 INFO [RS:2;ae3311ea5ccb:45793 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-15T21:59:06,717 INFO [RS:2;ae3311ea5ccb:45793 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-15T21:59:06,717 INFO [RS:2;ae3311ea5ccb:45793 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-15T21:59:06,718 INFO [RS:2;ae3311ea5ccb:45793 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-15T21:59:06,718 INFO [RS:2;ae3311ea5ccb:45793 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-15T21:59:06,718 INFO [RS:2;ae3311ea5ccb:45793 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-15T21:59:06,719 DEBUG [RS:2;ae3311ea5ccb:45793 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/ae3311ea5ccb:0, corePoolSize=1, maxPoolSize=1 2024-11-15T21:59:06,719 DEBUG [RS:2;ae3311ea5ccb:45793 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/ae3311ea5ccb:0, corePoolSize=1, maxPoolSize=1 2024-11-15T21:59:06,719 DEBUG [RS:2;ae3311ea5ccb:45793 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/ae3311ea5ccb:0, corePoolSize=1, maxPoolSize=1 2024-11-15T21:59:06,719 DEBUG [RS:2;ae3311ea5ccb:45793 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/ae3311ea5ccb:0, corePoolSize=1, maxPoolSize=1 2024-11-15T21:59:06,719 DEBUG [RS:2;ae3311ea5ccb:45793 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/ae3311ea5ccb:0, corePoolSize=1, maxPoolSize=1 2024-11-15T21:59:06,719 DEBUG [RS:2;ae3311ea5ccb:45793 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/ae3311ea5ccb:0, corePoolSize=2, maxPoolSize=2 2024-11-15T21:59:06,719 DEBUG [RS:2;ae3311ea5ccb:45793 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/ae3311ea5ccb:0, corePoolSize=1, maxPoolSize=1 2024-11-15T21:59:06,719 DEBUG [RS:2;ae3311ea5ccb:45793 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/ae3311ea5ccb:0, corePoolSize=1, maxPoolSize=1 2024-11-15T21:59:06,719 DEBUG [RS:2;ae3311ea5ccb:45793 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/ae3311ea5ccb:0, corePoolSize=1, maxPoolSize=1 2024-11-15T21:59:06,719 DEBUG [RS:2;ae3311ea5ccb:45793 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/ae3311ea5ccb:0, corePoolSize=1, maxPoolSize=1 2024-11-15T21:59:06,719 DEBUG [RS:2;ae3311ea5ccb:45793 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/ae3311ea5ccb:0, corePoolSize=1, maxPoolSize=1 2024-11-15T21:59:06,719 DEBUG [RS:2;ae3311ea5ccb:45793 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/ae3311ea5ccb:0, corePoolSize=1, maxPoolSize=1 2024-11-15T21:59:06,720 DEBUG [RS:2;ae3311ea5ccb:45793 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/ae3311ea5ccb:0, corePoolSize=3, maxPoolSize=3 2024-11-15T21:59:06,720 DEBUG [RS:2;ae3311ea5ccb:45793 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/ae3311ea5ccb:0, corePoolSize=3, maxPoolSize=3 2024-11-15T21:59:06,720 INFO [RS:2;ae3311ea5ccb:45793 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-15T21:59:06,720 INFO [RS:2;ae3311ea5ccb:45793 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-15T21:59:06,720 INFO [RS:2;ae3311ea5ccb:45793 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-15T21:59:06,720 INFO [RS:2;ae3311ea5ccb:45793 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-15T21:59:06,720 INFO [RS:2;ae3311ea5ccb:45793 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-15T21:59:06,720 INFO [RS:2;ae3311ea5ccb:45793 {}] hbase.ChoreService(168): Chore ScheduledChore name=ae3311ea5ccb,45793,1731707946654-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-15T21:59:06,726 INFO [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] handler.AssignRegionHandler(132): Open testBlockEvictionOnGracefulStop1731707944409,,1731707944413.d2cd4a927d1dc5c0d997e9855e15f923. 2024-11-15T21:59:06,726 DEBUG [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(7752): Opening region: {ENCODED => d2cd4a927d1dc5c0d997e9855e15f923, NAME => 'testBlockEvictionOnGracefulStop1731707944409,,1731707944413.d2cd4a927d1dc5c0d997e9855e15f923.', STARTKEY => '', ENDKEY => ''} 2024-11-15T21:59:06,727 DEBUG [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testBlockEvictionOnGracefulStop1731707944409 d2cd4a927d1dc5c0d997e9855e15f923 2024-11-15T21:59:06,727 DEBUG [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(898): Instantiated testBlockEvictionOnGracefulStop1731707944409,,1731707944413.d2cd4a927d1dc5c0d997e9855e15f923.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-15T21:59:06,727 DEBUG [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(7794): checking encryption for d2cd4a927d1dc5c0d997e9855e15f923 2024-11-15T21:59:06,727 DEBUG [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(7797): checking classloading for d2cd4a927d1dc5c0d997e9855e15f923 2024-11-15T21:59:06,728 INFO [StoreOpener-d2cd4a927d1dc5c0d997e9855e15f923-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family family of region d2cd4a927d1dc5c0d997e9855e15f923 2024-11-15T21:59:06,729 INFO [StoreOpener-d2cd4a927d1dc5c0d997e9855e15f923-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region d2cd4a927d1dc5c0d997e9855e15f923 columnFamilyName family 2024-11-15T21:59:06,729 DEBUG [StoreOpener-d2cd4a927d1dc5c0d997e9855e15f923-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T21:59:06,730 DEBUG [StoreFileOpener-d2cd4a927d1dc5c0d997e9855e15f923-family-1 {}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3be1b200-2a9c-651b-3f6b-ac964062b7d6/f4b02001-5f5c-c384-44d5-c547c79a7f1b/data/default/testBlockEvictionOnGracefulStop1731707944409/d2cd4a927d1dc5c0d997e9855e15f923/family/ebf6a2ec462e4af99422abc19ec14176, delay=930 ms 2024-11-15T21:59:06,732 DEBUG [StoreOpener-d2cd4a927d1dc5c0d997e9855e15f923-1 {}] regionserver.StoreEngine(278): loaded file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3be1b200-2a9c-651b-3f6b-ac964062b7d6/f4b02001-5f5c-c384-44d5-c547c79a7f1b/data/default/testBlockEvictionOnGracefulStop1731707944409/d2cd4a927d1dc5c0d997e9855e15f923/family/ebf6a2ec462e4af99422abc19ec14176 2024-11-15T21:59:06,732 INFO [StoreOpener-d2cd4a927d1dc5c0d997e9855e15f923-1 {}] regionserver.HStore(327): Store=d2cd4a927d1dc5c0d997e9855e15f923/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-15T21:59:06,732 DEBUG [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1038): replaying wal for d2cd4a927d1dc5c0d997e9855e15f923 2024-11-15T21:59:06,733 DEBUG [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3be1b200-2a9c-651b-3f6b-ac964062b7d6/f4b02001-5f5c-c384-44d5-c547c79a7f1b/data/default/testBlockEvictionOnGracefulStop1731707944409/d2cd4a927d1dc5c0d997e9855e15f923 2024-11-15T21:59:06,734 DEBUG [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3be1b200-2a9c-651b-3f6b-ac964062b7d6/f4b02001-5f5c-c384-44d5-c547c79a7f1b/data/default/testBlockEvictionOnGracefulStop1731707944409/d2cd4a927d1dc5c0d997e9855e15f923 2024-11-15T21:59:06,734 DEBUG [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1048): stopping wal replay for d2cd4a927d1dc5c0d997e9855e15f923 2024-11-15T21:59:06,734 DEBUG [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1060): Cleaning up temporary data for d2cd4a927d1dc5c0d997e9855e15f923 2024-11-15T21:59:06,736 DEBUG [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1093): writing seq id for d2cd4a927d1dc5c0d997e9855e15f923 2024-11-15T21:59:06,736 INFO [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1114): Opened d2cd4a927d1dc5c0d997e9855e15f923; next sequenceid=10; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=73248793, jitterRate=0.09149207174777985}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-15T21:59:06,737 DEBUG [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1122): Running coprocessor post-open hooks for d2cd4a927d1dc5c0d997e9855e15f923 2024-11-15T21:59:06,738 DEBUG [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1006): Region open journal for d2cd4a927d1dc5c0d997e9855e15f923: Running coprocessor pre-open hook at 1731707946727Writing region info on filesystem at 1731707946727Initializing all the Stores at 1731707946728 (+1 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731707946728Cleaning up temporary data from old regions at 1731707946734 (+6 ms)Running coprocessor post-open hooks at 1731707946737 (+3 ms)Region opened successfully at 1731707946738 (+1 ms) 2024-11-15T21:59:06,739 INFO [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegionServer(2236): Post open deploy tasks for testBlockEvictionOnGracefulStop1731707944409,,1731707944413.d2cd4a927d1dc5c0d997e9855e15f923., pid=9, masterSystemTime=1731707946717 2024-11-15T21:59:06,742 DEBUG [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegionServer(2266): Finished post open deploy task for testBlockEvictionOnGracefulStop1731707944409,,1731707944413.d2cd4a927d1dc5c0d997e9855e15f923. 2024-11-15T21:59:06,742 INFO [RS_OPEN_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] handler.AssignRegionHandler(153): Opened testBlockEvictionOnGracefulStop1731707944409,,1731707944413.d2cd4a927d1dc5c0d997e9855e15f923. 2024-11-15T21:59:06,743 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=8 updating hbase:meta row=d2cd4a927d1dc5c0d997e9855e15f923, regionState=OPEN, openSeqNum=10, regionLocation=ae3311ea5ccb,41593,1731707943200 2024-11-15T21:59:06,745 INFO [RS:2;ae3311ea5ccb:45793 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-15T21:59:06,745 INFO [RS:2;ae3311ea5ccb:45793 {}] hbase.ChoreService(168): Chore ScheduledChore name=ae3311ea5ccb,45793,1731707946654-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-15T21:59:06,745 INFO [RS:2;ae3311ea5ccb:45793 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-15T21:59:06,745 INFO [RS:2;ae3311ea5ccb:45793 {}] regionserver.Replication(171): ae3311ea5ccb,45793,1731707946654 started 2024-11-15T21:59:06,745 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-8-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=9, ppid=8, state=RUNNABLE, hasLock=false; OpenRegionProcedure d2cd4a927d1dc5c0d997e9855e15f923, server=ae3311ea5ccb,41593,1731707943200 because future has completed 2024-11-15T21:59:06,752 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=9, resume processing ppid=8 2024-11-15T21:59:06,753 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=9, ppid=8, state=SUCCESS, hasLock=false; OpenRegionProcedure d2cd4a927d1dc5c0d997e9855e15f923, server=ae3311ea5ccb,41593,1731707943200 in 185 msec 2024-11-15T21:59:06,755 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=8, resume processing ppid=7 2024-11-15T21:59:06,755 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=8, ppid=7, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1731707944409, region=d2cd4a927d1dc5c0d997e9855e15f923, ASSIGN in 344 msec 2024-11-15T21:59:06,755 INFO [PEWorker-4 {}] procedure.ServerCrashProcedure(291): removed crashed server ae3311ea5ccb,36535,1731707943240 after splitting done 2024-11-15T21:59:06,758 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=7, state=SUCCESS, hasLock=false; ServerCrashProcedure ae3311ea5ccb,36535,1731707943240, splitWal=true, meta=false in 358 msec 2024-11-15T21:59:06,766 INFO [RS:2;ae3311ea5ccb:45793 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-15T21:59:06,766 INFO [RS:2;ae3311ea5ccb:45793 {}] regionserver.HRegionServer(1482): Serving as ae3311ea5ccb,45793,1731707946654, RpcServer on ae3311ea5ccb/172.17.0.3:45793, sessionid=0x100f8692d830003 2024-11-15T21:59:06,766 DEBUG [RS:2;ae3311ea5ccb:45793 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-15T21:59:06,766 DEBUG [RS:2;ae3311ea5ccb:45793 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager ae3311ea5ccb,45793,1731707946654 2024-11-15T21:59:06,766 DEBUG [RS:2;ae3311ea5ccb:45793 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'ae3311ea5ccb,45793,1731707946654' 2024-11-15T21:59:06,766 DEBUG [RS:2;ae3311ea5ccb:45793 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-15T21:59:06,770 DEBUG [RS:2;ae3311ea5ccb:45793 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-15T21:59:06,771 DEBUG [RS:2;ae3311ea5ccb:45793 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-15T21:59:06,771 DEBUG [RS:2;ae3311ea5ccb:45793 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-15T21:59:06,771 DEBUG [RS:2;ae3311ea5ccb:45793 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager ae3311ea5ccb,45793,1731707946654 2024-11-15T21:59:06,771 DEBUG [RS:2;ae3311ea5ccb:45793 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'ae3311ea5ccb,45793,1731707946654' 2024-11-15T21:59:06,771 DEBUG [RS:2;ae3311ea5ccb:45793 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-15T21:59:06,772 DEBUG [RS:2;ae3311ea5ccb:45793 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-15T21:59:06,773 DEBUG [RS:2;ae3311ea5ccb:45793 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-15T21:59:06,773 INFO [RS:2;ae3311ea5ccb:45793 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-15T21:59:06,773 INFO [RS:2;ae3311ea5ccb:45793 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-15T21:59:06,874 INFO [RS:2;ae3311ea5ccb:45793 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-15T21:59:06,876 INFO [RS:2;ae3311ea5ccb:45793 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=ae3311ea5ccb%2C45793%2C1731707946654, suffix=, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3be1b200-2a9c-651b-3f6b-ac964062b7d6/f4b02001-5f5c-c384-44d5-c547c79a7f1b/WALs/ae3311ea5ccb,45793,1731707946654, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3be1b200-2a9c-651b-3f6b-ac964062b7d6/f4b02001-5f5c-c384-44d5-c547c79a7f1b/oldWALs, maxLogs=55 2024-11-15T21:59:06,911 INFO [RS:2;ae3311ea5ccb:45793 {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3be1b200-2a9c-651b-3f6b-ac964062b7d6/f4b02001-5f5c-c384-44d5-c547c79a7f1b/WALs/ae3311ea5ccb,45793,1731707946654/ae3311ea5ccb%2C45793%2C1731707946654.1731707946877 2024-11-15T21:59:06,911 DEBUG [RS:2;ae3311ea5ccb:45793 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-11-15T21:59:07,126 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta 2024-11-15T21:59:07,127 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_meta Metrics about Tables on a single HBase RegionServer 2024-11-15T21:59:07,128 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-15T21:59:07,128 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint Metrics about HBase RegionObservers 2024-11-15T21:59:07,129 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_master_table_store 2024-11-15T21:59:07,129 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_master_table_store Metrics about Tables on a single HBase RegionServer 2024-11-15T21:59:07,130 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testBlockEvictionOnGracefulStop1731707944409 2024-11-15T21:59:07,130 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testBlockEvictionOnGracefulStop1731707944409 Metrics about Tables on a single HBase RegionServer 2024-11-15T21:59:07,269 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster 2024-11-15T21:59:07,269 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-15T21:59:07,270 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79) at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611) at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.io.hfile.TestBlockEvictionOnRegionMovement.tearDown(TestBlockEvictionOnRegionMovement.java:174) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-15T21:59:07,270 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T21:59:07,270 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T21:59:07,270 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-15T21:59:07,270 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2024-11-15T21:59:07,270 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=203969093, stopped=false 2024-11-15T21:59:07,270 INFO [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=ae3311ea5ccb,42653,1731707943146 2024-11-15T21:59:07,273 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41593-0x100f8692d830001, quorum=127.0.0.1:54544, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-15T21:59:07,273 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45793-0x100f8692d830003, quorum=127.0.0.1:54544, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-15T21:59:07,273 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41593-0x100f8692d830001, quorum=127.0.0.1:54544, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-15T21:59:07,273 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45793-0x100f8692d830003, quorum=127.0.0.1:54544, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-15T21:59:07,273 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-15T21:59:07,273 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42653-0x100f8692d830000, quorum=127.0.0.1:54544, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-15T21:59:07,273 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42653-0x100f8692d830000, quorum=127.0.0.1:54544, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-15T21:59:07,273 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-15T21:59:07,273 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277) at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265) at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416) at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.io.hfile.TestBlockEvictionOnRegionMovement.tearDown(TestBlockEvictionOnRegionMovement.java:174) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-15T21:59:07,273 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T21:59:07,274 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server 'ae3311ea5ccb,41593,1731707943200' ***** 2024-11-15T21:59:07,274 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-11-15T21:59:07,274 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server 'ae3311ea5ccb,45793,1731707946654' ***** 2024-11-15T21:59:07,274 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-11-15T21:59:07,274 INFO [RS:0;ae3311ea5ccb:41593 {}] bucket.BucketCache(1738): Disabling cache 2024-11-15T21:59:07,275 INFO [RS:0;ae3311ea5ccb:41593 {}] bucket.BucketCache(1762): Shutdown bucket cache: IO persistent=true; path to write=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f706088d-a2ef-5fec-6a8f-fdcb2a34ea2f/bucket.persistence 2024-11-15T21:59:07,275 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:45793-0x100f8692d830003, quorum=127.0.0.1:54544, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-15T21:59:07,275 INFO [RS:2;ae3311ea5ccb:45793 {}] bucket.BucketCache(1738): Disabling cache 2024-11-15T21:59:07,275 INFO [RS:2;ae3311ea5ccb:45793 {}] bucket.BucketCache(1762): Shutdown bucket cache: IO persistent=true; path to write=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f706088d-a2ef-5fec-6a8f-fdcb2a34ea2f/bucket.persistence 2024-11-15T21:59:07,275 INFO [Time-limited test-BucketCacheWriter-0 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-0 exiting, cacheEnabled=false 2024-11-15T21:59:07,275 INFO [Time-limited test-BucketCacheWriter-2 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-2 exiting, cacheEnabled=false 2024-11-15T21:59:07,275 INFO [Time-limited test-BucketCacheWriter-1 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-1 exiting, cacheEnabled=false 2024-11-15T21:59:07,275 INFO [Time-limited test-BucketCacheWriter-0 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-0 exiting, cacheEnabled=false 2024-11-15T21:59:07,275 INFO [Time-limited test-BucketCacheWriter-1 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-1 exiting, cacheEnabled=false 2024-11-15T21:59:07,276 INFO [Time-limited test-BucketCacheWriter-2 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-2 exiting, cacheEnabled=false 2024-11-15T21:59:07,276 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:42653-0x100f8692d830000, quorum=127.0.0.1:54544, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-15T21:59:07,276 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:41593-0x100f8692d830001, quorum=127.0.0.1:54544, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-15T21:59:07,277 INFO [RS:0;ae3311ea5ccb:41593 {}] bucket.BucketCache(1768): Shutting down cache persister thread. 2024-11-15T21:59:07,277 WARN [bucket-cache-persister {}] bucket.BucketCachePersister(64): Interrupting BucketCachePersister thread. java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCachePersister.run(BucketCachePersister.java:45) ~[classes/:?] 2024-11-15T21:59:07,277 INFO [RS:2;ae3311ea5ccb:45793 {}] bucket.BucketCache(1768): Shutting down cache persister thread. 2024-11-15T21:59:07,278 WARN [bucket-cache-persister {}] bucket.BucketCachePersister(64): Interrupting BucketCachePersister thread. java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCachePersister.run(BucketCachePersister.java:45) ~[classes/:?] 2024-11-15T21:59:07,287 DEBUG [RS:0;ae3311ea5ccb:41593 {}] bucket.BucketCache(1487): Thread RS:0;ae3311ea5ccb:41593 started persisting bucket cache to file 2024-11-15T21:59:07,288 DEBUG [RS:0;ae3311ea5ccb:41593 {}] bucket.BucketCache(1494): Persist in new chunked persistence format. 2024-11-15T21:59:07,288 DEBUG [RS:2;ae3311ea5ccb:45793 {}] bucket.BucketCache(1487): Thread RS:2;ae3311ea5ccb:45793 started persisting bucket cache to file 2024-11-15T21:59:07,288 DEBUG [RS:0;ae3311ea5ccb:41593 {}] bucket.BucketCache(1673): persistToFile: before persisting backing map size: 0, fullycachedFiles size: 0, chunkSize: 10000 2024-11-15T21:59:07,288 DEBUG [RS:2;ae3311ea5ccb:45793 {}] bucket.BucketCache(1494): Persist in new chunked persistence format. 2024-11-15T21:59:07,288 DEBUG [RS:2;ae3311ea5ccb:45793 {}] bucket.BucketCache(1673): persistToFile: before persisting backing map size: 1, fullycachedFiles size: 0, chunkSize: 10000 2024-11-15T21:59:07,289 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-15T21:59:07,293 DEBUG [RS:0;ae3311ea5ccb:41593 {}] bucket.BucketCache(1680): persistToFile: after persisting backing map size: 0, fullycachedFiles size: 0 2024-11-15T21:59:07,293 DEBUG [RS:0;ae3311ea5ccb:41593 {}] bucket.BucketCache(1498): PersistToFile: after persisting backing map size: 0, fullycachedFiles size: 0, file name: bucket.persistence1731707947287 2024-11-15T21:59:07,293 DEBUG [RS:0;ae3311ea5ccb:41593 {}] bucket.BucketCache(1509): Thread RS:0;ae3311ea5ccb:41593 finished persisting bucket cache to file, renaming 2024-11-15T21:59:07,293 INFO [RS:0;ae3311ea5ccb:41593 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-15T21:59:07,293 DEBUG [RS:2;ae3311ea5ccb:45793 {}] bucket.BucketCache(1680): persistToFile: after persisting backing map size: 1, fullycachedFiles size: 0 2024-11-15T21:59:07,293 DEBUG [RS:2;ae3311ea5ccb:45793 {}] bucket.BucketCache(1498): PersistToFile: after persisting backing map size: 1, fullycachedFiles size: 0, file name: bucket.persistence1731707947288 2024-11-15T21:59:07,293 DEBUG [RS:2;ae3311ea5ccb:45793 {}] bucket.BucketCache(1509): Thread RS:2;ae3311ea5ccb:45793 finished persisting bucket cache to file, renaming 2024-11-15T21:59:07,293 INFO [RS:0;ae3311ea5ccb:41593 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-15T21:59:07,293 INFO [RS:0;ae3311ea5ccb:41593 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-15T21:59:07,294 INFO [RS:0;ae3311ea5ccb:41593 {}] regionserver.HRegionServer(3091): Received CLOSE for d2cd4a927d1dc5c0d997e9855e15f923 2024-11-15T21:59:07,294 INFO [RS:0;ae3311ea5ccb:41593 {}] regionserver.HRegionServer(959): stopping server ae3311ea5ccb,41593,1731707943200 2024-11-15T21:59:07,294 INFO [RS:0;ae3311ea5ccb:41593 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-15T21:59:07,294 INFO [RS:0;ae3311ea5ccb:41593 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;ae3311ea5ccb:41593. 2024-11-15T21:59:07,294 INFO [RS:2;ae3311ea5ccb:45793 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-15T21:59:07,294 DEBUG [RS:0;ae3311ea5ccb:41593 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-15T21:59:07,294 INFO [RS:2;ae3311ea5ccb:45793 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-15T21:59:07,294 DEBUG [RS:0;ae3311ea5ccb:41593 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T21:59:07,294 INFO [RS_CLOSE_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1662): Closing region testBlockEvictionOnGracefulStop1731707944409,,1731707944413.d2cd4a927d1dc5c0d997e9855e15f923. during a graceful stop, and cache persistence is on, so setting evict on close to false. 2024-11-15T21:59:07,294 INFO [RS:2;ae3311ea5ccb:45793 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-15T21:59:07,294 INFO [RS:2;ae3311ea5ccb:45793 {}] regionserver.HRegionServer(959): stopping server ae3311ea5ccb,45793,1731707946654 2024-11-15T21:59:07,294 INFO [RS:2;ae3311ea5ccb:45793 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-15T21:59:07,294 INFO [RS:0;ae3311ea5ccb:41593 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-15T21:59:07,294 INFO [RS:0;ae3311ea5ccb:41593 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-15T21:59:07,294 DEBUG [RS_CLOSE_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing d2cd4a927d1dc5c0d997e9855e15f923, disabling compactions & flushes 2024-11-15T21:59:07,294 INFO [RS:0;ae3311ea5ccb:41593 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-15T21:59:07,295 INFO [RS:0;ae3311ea5ccb:41593 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-11-15T21:59:07,295 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-15T21:59:07,294 INFO [RS_CLOSE_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testBlockEvictionOnGracefulStop1731707944409,,1731707944413.d2cd4a927d1dc5c0d997e9855e15f923. 2024-11-15T21:59:07,295 INFO [RS:2;ae3311ea5ccb:45793 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:2;ae3311ea5ccb:45793. 2024-11-15T21:59:07,295 DEBUG [RS_CLOSE_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testBlockEvictionOnGracefulStop1731707944409,,1731707944413.d2cd4a927d1dc5c0d997e9855e15f923. 2024-11-15T21:59:07,295 DEBUG [RS_CLOSE_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testBlockEvictionOnGracefulStop1731707944409,,1731707944413.d2cd4a927d1dc5c0d997e9855e15f923. after waiting 0 ms 2024-11-15T21:59:07,295 DEBUG [RS_CLOSE_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testBlockEvictionOnGracefulStop1731707944409,,1731707944413.d2cd4a927d1dc5c0d997e9855e15f923. 2024-11-15T21:59:07,295 DEBUG [RS:2;ae3311ea5ccb:45793 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-15T21:59:07,295 DEBUG [RS:2;ae3311ea5ccb:45793 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T21:59:07,296 INFO [RS:2;ae3311ea5ccb:45793 {}] regionserver.HRegionServer(976): stopping server ae3311ea5ccb,45793,1731707946654; all regions closed. 2024-11-15T21:59:07,297 INFO [regionserver/ae3311ea5ccb:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-15T21:59:07,297 INFO [RS:0;ae3311ea5ccb:41593 {}] regionserver.HRegionServer(1321): Waiting on 2 regions to close 2024-11-15T21:59:07,297 DEBUG [RS:0;ae3311ea5ccb:41593 {}] regionserver.HRegionServer(1325): Online Regions={d2cd4a927d1dc5c0d997e9855e15f923=testBlockEvictionOnGracefulStop1731707944409,,1731707944413.d2cd4a927d1dc5c0d997e9855e15f923., 1588230740=hbase:meta,,1.1588230740} 2024-11-15T21:59:07,297 DEBUG [RS:0;ae3311ea5ccb:41593 {}] regionserver.HRegionServer(1351): Waiting on 1588230740, d2cd4a927d1dc5c0d997e9855e15f923 2024-11-15T21:59:07,300 INFO [RS_CLOSE_META-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1662): Closing region hbase:meta,,1.1588230740 during a graceful stop, and cache persistence is on, so setting evict on close to false. 2024-11-15T21:59:07,300 DEBUG [RS_CLOSE_META-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-15T21:59:07,300 INFO [RS_CLOSE_META-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-15T21:59:07,300 DEBUG [RS_CLOSE_META-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-15T21:59:07,300 DEBUG [RS_CLOSE_META-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-15T21:59:07,300 DEBUG [RS_CLOSE_META-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-15T21:59:07,300 INFO [RS_CLOSE_META-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=2.96 KB heapSize=5.58 KB 2024-11-15T21:59:07,305 DEBUG [StoreFileCloser-d2cd4a927d1dc5c0d997e9855e15f923-family-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3be1b200-2a9c-651b-3f6b-ac964062b7d6/f4b02001-5f5c-c384-44d5-c547c79a7f1b/data/default/testBlockEvictionOnGracefulStop1731707944409/d2cd4a927d1dc5c0d997e9855e15f923/family/ebf6a2ec462e4af99422abc19ec14176 2024-11-15T21:59:07,315 DEBUG [RS_CLOSE_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3be1b200-2a9c-651b-3f6b-ac964062b7d6/f4b02001-5f5c-c384-44d5-c547c79a7f1b/data/default/testBlockEvictionOnGracefulStop1731707944409/d2cd4a927d1dc5c0d997e9855e15f923/recovered.edits/12.seqid, newMaxSeqId=12, maxSeqId=9 2024-11-15T21:59:07,316 INFO [RS_CLOSE_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testBlockEvictionOnGracefulStop1731707944409,,1731707944413.d2cd4a927d1dc5c0d997e9855e15f923. 2024-11-15T21:59:07,316 DEBUG [RS_CLOSE_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for d2cd4a927d1dc5c0d997e9855e15f923: Waiting for close lock at 1731707947294Running coprocessor pre-close hooks at 1731707947294Disabling compacts and flushes for region at 1731707947294Disabling writes for close at 1731707947295 (+1 ms)Writing region close event to WAL at 1731707947306 (+11 ms)Running coprocessor post-close hooks at 1731707947316 (+10 ms)Closed at 1731707947316 2024-11-15T21:59:07,316 DEBUG [RS_CLOSE_REGION-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testBlockEvictionOnGracefulStop1731707944409,,1731707944413.d2cd4a927d1dc5c0d997e9855e15f923. 2024-11-15T21:59:07,320 DEBUG [RS_CLOSE_META-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 16220ce7bffc41d69d71930f0fa89e50 2024-11-15T21:59:07,320 DEBUG [RS:2;ae3311ea5ccb:45793 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3be1b200-2a9c-651b-3f6b-ac964062b7d6/f4b02001-5f5c-c384-44d5-c547c79a7f1b/oldWALs 2024-11-15T21:59:07,321 INFO [RS:2;ae3311ea5ccb:45793 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL ae3311ea5ccb%2C45793%2C1731707946654:(num 1731707946877) 2024-11-15T21:59:07,321 DEBUG [RS:2;ae3311ea5ccb:45793 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T21:59:07,321 INFO [RS:2;ae3311ea5ccb:45793 {}] regionserver.LeaseManager(133): Closed leases 2024-11-15T21:59:07,321 INFO [RS:2;ae3311ea5ccb:45793 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-15T21:59:07,321 INFO [RS:2;ae3311ea5ccb:45793 {}] hbase.ChoreService(370): Chore service for: regionserver/ae3311ea5ccb:0 had [ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-11-15T21:59:07,322 INFO [RS:2;ae3311ea5ccb:45793 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-15T21:59:07,322 INFO [RS:2;ae3311ea5ccb:45793 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-15T21:59:07,322 INFO [regionserver/ae3311ea5ccb:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-15T21:59:07,322 INFO [RS:2;ae3311ea5ccb:45793 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-15T21:59:07,322 INFO [RS:2;ae3311ea5ccb:45793 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-15T21:59:07,322 INFO [RS:2;ae3311ea5ccb:45793 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.3:45793 2024-11-15T21:59:07,323 INFO [regionserver/ae3311ea5ccb:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-15T21:59:07,327 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42653-0x100f8692d830000, quorum=127.0.0.1:54544, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-15T21:59:07,328 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45793-0x100f8692d830003, quorum=127.0.0.1:54544, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/ae3311ea5ccb,45793,1731707946654 2024-11-15T21:59:07,328 INFO [RS:2;ae3311ea5ccb:45793 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-15T21:59:07,333 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [ae3311ea5ccb,45793,1731707946654] 2024-11-15T21:59:07,341 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/ae3311ea5ccb,45793,1731707946654 already deleted, retry=false 2024-11-15T21:59:07,341 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; ae3311ea5ccb,45793,1731707946654 expired; onlineServers=1 2024-11-15T21:59:07,341 DEBUG [RS_CLOSE_META-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3be1b200-2a9c-651b-3f6b-ac964062b7d6/f4b02001-5f5c-c384-44d5-c547c79a7f1b/data/hbase/meta/1588230740/.tmp/info/16220ce7bffc41d69d71930f0fa89e50 is 209, key is testBlockEvictionOnGracefulStop1731707944409,,1731707944413.d2cd4a927d1dc5c0d997e9855e15f923./info:regioninfo/1731707946743/Put/seqid=0 2024-11-15T21:59:07,342 INFO [RS_CLOSE_META-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=2.74 KB at sequenceid=13 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3be1b200-2a9c-651b-3f6b-ac964062b7d6/f4b02001-5f5c-c384-44d5-c547c79a7f1b/data/hbase/meta/1588230740/.tmp/info/16220ce7bffc41d69d71930f0fa89e50 2024-11-15T21:59:07,365 DEBUG [RS_CLOSE_META-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 0eda1bd0b4af4f36b26633792579b50f 2024-11-15T21:59:07,382 DEBUG [RS_CLOSE_META-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3be1b200-2a9c-651b-3f6b-ac964062b7d6/f4b02001-5f5c-c384-44d5-c547c79a7f1b/data/hbase/meta/1588230740/.tmp/ns/0eda1bd0b4af4f36b26633792579b50f is 43, key is default/ns:d/1731707944341/Put/seqid=0 2024-11-15T21:59:07,383 INFO [RS_CLOSE_META-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=74 B at sequenceid=13 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3be1b200-2a9c-651b-3f6b-ac964062b7d6/f4b02001-5f5c-c384-44d5-c547c79a7f1b/data/hbase/meta/1588230740/.tmp/ns/0eda1bd0b4af4f36b26633792579b50f 2024-11-15T21:59:07,396 DEBUG [RS_CLOSE_META-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFile$WriterFactory(308): Unable to set drop behind on dde9013aec8f4a69985673162fe3ff46 2024-11-15T21:59:07,416 DEBUG [RS_CLOSE_META-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3be1b200-2a9c-651b-3f6b-ac964062b7d6/f4b02001-5f5c-c384-44d5-c547c79a7f1b/data/hbase/meta/1588230740/.tmp/table/dde9013aec8f4a69985673162fe3ff46 is 80, key is testBlockEvictionOnGracefulStop1731707944409/table:state/1731707944840/Put/seqid=0 2024-11-15T21:59:07,417 INFO [RS_CLOSE_META-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=152 B at sequenceid=13 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3be1b200-2a9c-651b-3f6b-ac964062b7d6/f4b02001-5f5c-c384-44d5-c547c79a7f1b/data/hbase/meta/1588230740/.tmp/table/dde9013aec8f4a69985673162fe3ff46 2024-11-15T21:59:07,419 DEBUG [RS_CLOSE_META-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3be1b200-2a9c-651b-3f6b-ac964062b7d6/f4b02001-5f5c-c384-44d5-c547c79a7f1b/data/hbase/meta/1588230740/.tmp/info/16220ce7bffc41d69d71930f0fa89e50 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3be1b200-2a9c-651b-3f6b-ac964062b7d6/f4b02001-5f5c-c384-44d5-c547c79a7f1b/data/hbase/meta/1588230740/info/16220ce7bffc41d69d71930f0fa89e50 2024-11-15T21:59:07,421 DEBUG [RS_CLOSE_META-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_CLOSE_META}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3be1b200-2a9c-651b-3f6b-ac964062b7d6/f4b02001-5f5c-c384-44d5-c547c79a7f1b/data/hbase/meta/1588230740/info/16220ce7bffc41d69d71930f0fa89e50, delay=964 ms 2024-11-15T21:59:07,422 INFO [RS_CLOSE_META-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3be1b200-2a9c-651b-3f6b-ac964062b7d6/f4b02001-5f5c-c384-44d5-c547c79a7f1b/data/hbase/meta/1588230740/info/16220ce7bffc41d69d71930f0fa89e50, entries=14, sequenceid=13, filesize=7.5 K 2024-11-15T21:59:07,422 DEBUG [RS_CLOSE_META-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3be1b200-2a9c-651b-3f6b-ac964062b7d6/f4b02001-5f5c-c384-44d5-c547c79a7f1b/data/hbase/meta/1588230740/.tmp/ns/0eda1bd0b4af4f36b26633792579b50f as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3be1b200-2a9c-651b-3f6b-ac964062b7d6/f4b02001-5f5c-c384-44d5-c547c79a7f1b/data/hbase/meta/1588230740/ns/0eda1bd0b4af4f36b26633792579b50f 2024-11-15T21:59:07,423 DEBUG [RS_CLOSE_META-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_CLOSE_META}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3be1b200-2a9c-651b-3f6b-ac964062b7d6/f4b02001-5f5c-c384-44d5-c547c79a7f1b/data/hbase/meta/1588230740/ns/0eda1bd0b4af4f36b26633792579b50f, delay=981 ms 2024-11-15T21:59:07,424 INFO [RS_CLOSE_META-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3be1b200-2a9c-651b-3f6b-ac964062b7d6/f4b02001-5f5c-c384-44d5-c547c79a7f1b/data/hbase/meta/1588230740/ns/0eda1bd0b4af4f36b26633792579b50f, entries=2, sequenceid=13, filesize=5.0 K 2024-11-15T21:59:07,425 DEBUG [RS_CLOSE_META-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3be1b200-2a9c-651b-3f6b-ac964062b7d6/f4b02001-5f5c-c384-44d5-c547c79a7f1b/data/hbase/meta/1588230740/.tmp/table/dde9013aec8f4a69985673162fe3ff46 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3be1b200-2a9c-651b-3f6b-ac964062b7d6/f4b02001-5f5c-c384-44d5-c547c79a7f1b/data/hbase/meta/1588230740/table/dde9013aec8f4a69985673162fe3ff46 2024-11-15T21:59:07,426 DEBUG [RS_CLOSE_META-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_CLOSE_META}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3be1b200-2a9c-651b-3f6b-ac964062b7d6/f4b02001-5f5c-c384-44d5-c547c79a7f1b/data/hbase/meta/1588230740/table/dde9013aec8f4a69985673162fe3ff46, delay=932 ms 2024-11-15T21:59:07,427 INFO [RS_CLOSE_META-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3be1b200-2a9c-651b-3f6b-ac964062b7d6/f4b02001-5f5c-c384-44d5-c547c79a7f1b/data/hbase/meta/1588230740/table/dde9013aec8f4a69985673162fe3ff46, entries=2, sequenceid=13, filesize=5.3 K 2024-11-15T21:59:07,428 INFO [RS_CLOSE_META-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~2.96 KB/3035, heapSize ~5.28 KB/5408, currentSize=0 B/0 for 1588230740 in 128ms, sequenceid=13, compaction requested=false 2024-11-15T21:59:07,428 DEBUG [StoreFileCloser-1588230740-info-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3be1b200-2a9c-651b-3f6b-ac964062b7d6/f4b02001-5f5c-c384-44d5-c547c79a7f1b/data/hbase/meta/1588230740/info/16220ce7bffc41d69d71930f0fa89e50 2024-11-15T21:59:07,429 DEBUG [StoreFileCloser-1588230740-ns-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3be1b200-2a9c-651b-3f6b-ac964062b7d6/f4b02001-5f5c-c384-44d5-c547c79a7f1b/data/hbase/meta/1588230740/ns/0eda1bd0b4af4f36b26633792579b50f 2024-11-15T21:59:07,430 DEBUG [StoreFileCloser-1588230740-table-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3be1b200-2a9c-651b-3f6b-ac964062b7d6/f4b02001-5f5c-c384-44d5-c547c79a7f1b/data/hbase/meta/1588230740/table/dde9013aec8f4a69985673162fe3ff46 2024-11-15T21:59:07,434 DEBUG [RS_CLOSE_META-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3be1b200-2a9c-651b-3f6b-ac964062b7d6/f4b02001-5f5c-c384-44d5-c547c79a7f1b/data/hbase/meta/1588230740/recovered.edits/16.seqid, newMaxSeqId=16, maxSeqId=1 2024-11-15T21:59:07,435 DEBUG [RS_CLOSE_META-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-15T21:59:07,435 INFO [RS_CLOSE_META-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-15T21:59:07,435 DEBUG [RS_CLOSE_META-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1731707947300Running coprocessor pre-close hooks at 1731707947300Disabling compacts and flushes for region at 1731707947300Disabling writes for close at 1731707947300Obtaining lock to block concurrent updates at 1731707947301 (+1 ms)Preparing flush snapshotting stores in 1588230740 at 1731707947301Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=3035, getHeapSize=5648, getOffHeapSize=0, getCellsCount=22 at 1731707947301Flushing stores of hbase:meta,,1.1588230740 at 1731707947301Flushing 1588230740/info: creating writer at 1731707947302 (+1 ms)Flushing 1588230740/info: appending metadata at 1731707947341 (+39 ms)Flushing 1588230740/info: closing flushed file at 1731707947341Flushing 1588230740/ns: creating writer at 1731707947345 (+4 ms)Flushing 1588230740/ns: appending metadata at 1731707947381 (+36 ms)Flushing 1588230740/ns: closing flushed file at 1731707947381Flushing 1588230740/table: creating writer at 1731707947386 (+5 ms)Flushing 1588230740/table: appending metadata at 1731707947415 (+29 ms)Flushing 1588230740/table: closing flushed file at 1731707947415Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@2898878e: reopening flushed file at 1731707947419 (+4 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5e0f1faf: reopening flushed file at 1731707947422 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@74abf7c0: reopening flushed file at 1731707947424 (+2 ms)Finished flush of dataSize ~2.96 KB/3035, heapSize ~5.28 KB/5408, currentSize=0 B/0 for 1588230740 in 128ms, sequenceid=13, compaction requested=false at 1731707947428 (+4 ms)Writing region close event to WAL at 1731707947430 (+2 ms)Running coprocessor post-close hooks at 1731707947435 (+5 ms)Closed at 1731707947435 2024-11-15T21:59:07,435 DEBUG [RS_CLOSE_META-regionserver/ae3311ea5ccb:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-11-15T21:59:07,438 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45793-0x100f8692d830003, quorum=127.0.0.1:54544, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-15T21:59:07,438 INFO [RS:2;ae3311ea5ccb:45793 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-15T21:59:07,438 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45793-0x100f8692d830003, quorum=127.0.0.1:54544, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-15T21:59:07,438 INFO [RS:2;ae3311ea5ccb:45793 {}] regionserver.HRegionServer(1031): Exiting; stopping=ae3311ea5ccb,45793,1731707946654; zookeeper connection closed. 2024-11-15T21:59:07,438 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@3a7ee9c5 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@3a7ee9c5 2024-11-15T21:59:07,498 INFO [RS:0;ae3311ea5ccb:41593 {}] regionserver.HRegionServer(976): stopping server ae3311ea5ccb,41593,1731707943200; all regions closed. 2024-11-15T21:59:07,500 DEBUG [RS:0;ae3311ea5ccb:41593 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3be1b200-2a9c-651b-3f6b-ac964062b7d6/f4b02001-5f5c-c384-44d5-c547c79a7f1b/oldWALs 2024-11-15T21:59:07,500 INFO [RS:0;ae3311ea5ccb:41593 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL ae3311ea5ccb%2C41593%2C1731707943200.meta:.meta(num 1731707944284) 2024-11-15T21:59:07,501 DEBUG [RS:0;ae3311ea5ccb:41593 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3be1b200-2a9c-651b-3f6b-ac964062b7d6/f4b02001-5f5c-c384-44d5-c547c79a7f1b/oldWALs 2024-11-15T21:59:07,501 INFO [RS:0;ae3311ea5ccb:41593 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL ae3311ea5ccb%2C41593%2C1731707943200:(num 1731707943936) 2024-11-15T21:59:07,501 DEBUG [RS:0;ae3311ea5ccb:41593 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T21:59:07,501 INFO [RS:0;ae3311ea5ccb:41593 {}] regionserver.LeaseManager(133): Closed leases 2024-11-15T21:59:07,501 INFO [RS:0;ae3311ea5ccb:41593 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-15T21:59:07,502 INFO [RS:0;ae3311ea5ccb:41593 {}] hbase.ChoreService(370): Chore service for: regionserver/ae3311ea5ccb:0 had [ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-11-15T21:59:07,502 INFO [RS:0;ae3311ea5ccb:41593 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-15T21:59:07,502 INFO [regionserver/ae3311ea5ccb:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-15T21:59:07,502 INFO [RS:0;ae3311ea5ccb:41593 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.3:41593 2024-11-15T21:59:07,508 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41593-0x100f8692d830001, quorum=127.0.0.1:54544, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/ae3311ea5ccb,41593,1731707943200 2024-11-15T21:59:07,508 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42653-0x100f8692d830000, quorum=127.0.0.1:54544, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-15T21:59:07,508 INFO [RS:0;ae3311ea5ccb:41593 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-15T21:59:07,509 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [ae3311ea5ccb,41593,1731707943200] 2024-11-15T21:59:07,514 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/ae3311ea5ccb,41593,1731707943200 already deleted, retry=false 2024-11-15T21:59:07,514 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; ae3311ea5ccb,41593,1731707943200 expired; onlineServers=0 2024-11-15T21:59:07,514 INFO [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master 'ae3311ea5ccb,42653,1731707943146' ***** 2024-11-15T21:59:07,514 INFO [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0 2024-11-15T21:59:07,515 INFO [M:0;ae3311ea5ccb:42653 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-15T21:59:07,515 INFO [M:0;ae3311ea5ccb:42653 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-15T21:59:07,515 DEBUG [M:0;ae3311ea5ccb:42653 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2024-11-15T21:59:07,515 DEBUG [M:0;ae3311ea5ccb:42653 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2024-11-15T21:59:07,515 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2024-11-15T21:59:07,515 DEBUG [master/ae3311ea5ccb:0:becomeActiveMaster-HFileCleaner.large.0-1731707943652 {}] cleaner.HFileCleaner(306): Exit Thread[master/ae3311ea5ccb:0:becomeActiveMaster-HFileCleaner.large.0-1731707943652,5,FailOnTimeoutGroup] 2024-11-15T21:59:07,515 DEBUG [master/ae3311ea5ccb:0:becomeActiveMaster-HFileCleaner.small.0-1731707943652 {}] cleaner.HFileCleaner(306): Exit Thread[master/ae3311ea5ccb:0:becomeActiveMaster-HFileCleaner.small.0-1731707943652,5,FailOnTimeoutGroup] 2024-11-15T21:59:07,515 INFO [M:0;ae3311ea5ccb:42653 {}] hbase.ChoreService(370): Chore service for: master/ae3311ea5ccb:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown 2024-11-15T21:59:07,515 INFO [M:0;ae3311ea5ccb:42653 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-15T21:59:07,515 DEBUG [M:0;ae3311ea5ccb:42653 {}] master.HMaster(1795): Stopping service threads 2024-11-15T21:59:07,515 INFO [M:0;ae3311ea5ccb:42653 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2024-11-15T21:59:07,516 INFO [M:0;ae3311ea5ccb:42653 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-15T21:59:07,516 INFO [M:0;ae3311ea5ccb:42653 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-15T21:59:07,516 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2024-11-15T21:59:07,517 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42653-0x100f8692d830000, quorum=127.0.0.1:54544, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-11-15T21:59:07,517 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42653-0x100f8692d830000, quorum=127.0.0.1:54544, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-15T21:59:07,517 DEBUG [M:0;ae3311ea5ccb:42653 {}] zookeeper.ZKUtil(347): master:42653-0x100f8692d830000, quorum=127.0.0.1:54544, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error) 2024-11-15T21:59:07,517 WARN [M:0;ae3311ea5ccb:42653 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null 2024-11-15T21:59:07,518 INFO [M:0;ae3311ea5ccb:42653 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3be1b200-2a9c-651b-3f6b-ac964062b7d6/f4b02001-5f5c-c384-44d5-c547c79a7f1b/.lastflushedseqids 2024-11-15T21:59:07,525 INFO [M:0;ae3311ea5ccb:42653 {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-15T21:59:07,525 INFO [M:0;ae3311ea5ccb:42653 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2024-11-15T21:59:07,525 DEBUG [M:0;ae3311ea5ccb:42653 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-15T21:59:07,525 INFO [M:0;ae3311ea5ccb:42653 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-15T21:59:07,525 DEBUG [M:0;ae3311ea5ccb:42653 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-15T21:59:07,525 DEBUG [M:0;ae3311ea5ccb:42653 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-15T21:59:07,525 DEBUG [M:0;ae3311ea5ccb:42653 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-15T21:59:07,525 INFO [M:0;ae3311ea5ccb:42653 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=32.83 KB heapSize=41.08 KB 2024-11-15T21:59:07,535 DEBUG [M:0;ae3311ea5ccb:42653 {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 4242d2c8e1aa4332afb1c8408fc46647 2024-11-15T21:59:07,549 DEBUG [M:0;ae3311ea5ccb:42653 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3be1b200-2a9c-651b-3f6b-ac964062b7d6/f4b02001-5f5c-c384-44d5-c547c79a7f1b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/4242d2c8e1aa4332afb1c8408fc46647 is 82, key is hbase:meta,,1/info:regioninfo/1731707944322/Put/seqid=0 2024-11-15T21:59:07,549 INFO [M:0;ae3311ea5ccb:42653 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=504 B at sequenceid=83 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3be1b200-2a9c-651b-3f6b-ac964062b7d6/f4b02001-5f5c-c384-44d5-c547c79a7f1b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/4242d2c8e1aa4332afb1c8408fc46647 2024-11-15T21:59:07,557 DEBUG [M:0;ae3311ea5ccb:42653 {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 74361f6f1e2a4464862ae2031d87f831 2024-11-15T21:59:07,571 DEBUG [M:0;ae3311ea5ccb:42653 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3be1b200-2a9c-651b-3f6b-ac964062b7d6/f4b02001-5f5c-c384-44d5-c547c79a7f1b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/74361f6f1e2a4464862ae2031d87f831 is 782, key is \x00\x00\x00\x00\x00\x00\x00\x04/proc:d/1731707944846/Put/seqid=0 2024-11-15T21:59:07,572 INFO [M:0;ae3311ea5ccb:42653 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=32.09 KB at sequenceid=83 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3be1b200-2a9c-651b-3f6b-ac964062b7d6/f4b02001-5f5c-c384-44d5-c547c79a7f1b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/74361f6f1e2a4464862ae2031d87f831 2024-11-15T21:59:07,580 DEBUG [M:0;ae3311ea5ccb:42653 {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 6ac4982a5b7d48989a23b675b67c78a6 2024-11-15T21:59:07,594 DEBUG [M:0;ae3311ea5ccb:42653 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3be1b200-2a9c-651b-3f6b-ac964062b7d6/f4b02001-5f5c-c384-44d5-c547c79a7f1b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/6ac4982a5b7d48989a23b675b67c78a6 is 69, key is ae3311ea5ccb,41593,1731707943200/rs:state/1731707943729/Put/seqid=0 2024-11-15T21:59:07,595 INFO [M:0;ae3311ea5ccb:42653 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=249 B at sequenceid=83 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3be1b200-2a9c-651b-3f6b-ac964062b7d6/f4b02001-5f5c-c384-44d5-c547c79a7f1b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/6ac4982a5b7d48989a23b675b67c78a6 2024-11-15T21:59:07,597 INFO [M:0;ae3311ea5ccb:42653 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 6ac4982a5b7d48989a23b675b67c78a6 2024-11-15T21:59:07,597 DEBUG [M:0;ae3311ea5ccb:42653 {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3be1b200-2a9c-651b-3f6b-ac964062b7d6/f4b02001-5f5c-c384-44d5-c547c79a7f1b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/4242d2c8e1aa4332afb1c8408fc46647 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3be1b200-2a9c-651b-3f6b-ac964062b7d6/f4b02001-5f5c-c384-44d5-c547c79a7f1b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/4242d2c8e1aa4332afb1c8408fc46647 2024-11-15T21:59:07,599 INFO [M:0;ae3311ea5ccb:42653 {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3be1b200-2a9c-651b-3f6b-ac964062b7d6/f4b02001-5f5c-c384-44d5-c547c79a7f1b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/4242d2c8e1aa4332afb1c8408fc46647, entries=8, sequenceid=83, filesize=5.5 K 2024-11-15T21:59:07,599 DEBUG [M:0;ae3311ea5ccb:42653 {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3be1b200-2a9c-651b-3f6b-ac964062b7d6/f4b02001-5f5c-c384-44d5-c547c79a7f1b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/74361f6f1e2a4464862ae2031d87f831 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3be1b200-2a9c-651b-3f6b-ac964062b7d6/f4b02001-5f5c-c384-44d5-c547c79a7f1b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/74361f6f1e2a4464862ae2031d87f831 2024-11-15T21:59:07,601 INFO [M:0;ae3311ea5ccb:42653 {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3be1b200-2a9c-651b-3f6b-ac964062b7d6/f4b02001-5f5c-c384-44d5-c547c79a7f1b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/74361f6f1e2a4464862ae2031d87f831, entries=9, sequenceid=83, filesize=6.4 K 2024-11-15T21:59:07,601 DEBUG [M:0;ae3311ea5ccb:42653 {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3be1b200-2a9c-651b-3f6b-ac964062b7d6/f4b02001-5f5c-c384-44d5-c547c79a7f1b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/6ac4982a5b7d48989a23b675b67c78a6 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3be1b200-2a9c-651b-3f6b-ac964062b7d6/f4b02001-5f5c-c384-44d5-c547c79a7f1b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/6ac4982a5b7d48989a23b675b67c78a6 2024-11-15T21:59:07,603 INFO [M:0;ae3311ea5ccb:42653 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 6ac4982a5b7d48989a23b675b67c78a6 2024-11-15T21:59:07,603 INFO [M:0;ae3311ea5ccb:42653 {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3be1b200-2a9c-651b-3f6b-ac964062b7d6/f4b02001-5f5c-c384-44d5-c547c79a7f1b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/6ac4982a5b7d48989a23b675b67c78a6, entries=3, sequenceid=83, filesize=5.3 K 2024-11-15T21:59:07,604 INFO [M:0;ae3311ea5ccb:42653 {}] regionserver.HRegion(3140): Finished flush of dataSize ~32.83 KB/33614, heapSize ~40.78 KB/41760, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 79ms, sequenceid=83, compaction requested=false 2024-11-15T21:59:07,606 INFO [M:0;ae3311ea5ccb:42653 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-15T21:59:07,606 DEBUG [M:0;ae3311ea5ccb:42653 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1731707947525Disabling compacts and flushes for region at 1731707947525Disabling writes for close at 1731707947525Obtaining lock to block concurrent updates at 1731707947525Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1731707947525Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=33614, getHeapSize=42000, getOffHeapSize=0, getCellsCount=98 at 1731707947526 (+1 ms)Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1731707947526Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1731707947526Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1731707947548 (+22 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1731707947548Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1731707947551 (+3 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1731707947571 (+20 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1731707947571Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1731707947574 (+3 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1731707947594 (+20 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1731707947594Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@7e57b6f8: reopening flushed file at 1731707947597 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5db46578: reopening flushed file at 1731707947599 (+2 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5eda44d3: reopening flushed file at 1731707947601 (+2 ms)Finished flush of dataSize ~32.83 KB/33614, heapSize ~40.78 KB/41760, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 79ms, sequenceid=83, compaction requested=false at 1731707947604 (+3 ms)Writing region close event to WAL at 1731707947605 (+1 ms)Closed at 1731707947605 2024-11-15T21:59:07,607 INFO [M:0;ae3311ea5ccb:42653 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down. 2024-11-15T21:59:07,607 INFO [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-15T21:59:07,607 INFO [M:0;ae3311ea5ccb:42653 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.3:42653 2024-11-15T21:59:07,607 INFO [M:0;ae3311ea5ccb:42653 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-15T21:59:07,614 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41593-0x100f8692d830001, quorum=127.0.0.1:54544, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-15T21:59:07,614 INFO [RS:0;ae3311ea5ccb:41593 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-15T21:59:07,614 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41593-0x100f8692d830001, quorum=127.0.0.1:54544, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-15T21:59:07,614 INFO [RS:0;ae3311ea5ccb:41593 {}] regionserver.HRegionServer(1031): Exiting; stopping=ae3311ea5ccb,41593,1731707943200; zookeeper connection closed. 2024-11-15T21:59:07,614 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@39f5940 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@39f5940 2024-11-15T21:59:07,614 INFO [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 3 regionserver(s) complete 2024-11-15T21:59:07,710 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42653-0x100f8692d830000, quorum=127.0.0.1:54544, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-15T21:59:07,710 INFO [M:0;ae3311ea5ccb:42653 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-15T21:59:07,710 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42653-0x100f8692d830000, quorum=127.0.0.1:54544, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-15T21:59:07,713 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers 2024-11-15T21:59:07,715 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down 2024-11-15T21:59:07,715 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-15T21:59:07,723 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: io.hfile.TestBlockEvictionOnRegionMovement#testBlockEvictionOnGracefulStop Thread=72 (was 51) Potentially hanging thread: HMaster-EventLoopGroup-6-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-10 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-6 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HMaster-EventLoopGroup-6-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-7-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HMaster-EventLoopGroup-6-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-8-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-7-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-8 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-9-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-9-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-8-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-9 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-7 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-8-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-7-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=382 (was 346) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=920 (was 939), ProcessCount=11 (was 11), AvailableMemoryMB=4894 (was 4180) - AvailableMemoryMB LEAK? -