2024-11-28 06:56:38,481 main DEBUG Apache Log4j Core 2.17.2 initializing configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@15a34df2 2024-11-28 06:56:38,514 main DEBUG Took 0.030049 seconds to load 1 plugins from package org.apache.hadoop.hbase.logging 2024-11-28 06:56:38,514 main DEBUG PluginManager 'Core' found 129 plugins 2024-11-28 06:56:38,515 main DEBUG PluginManager 'Level' found 0 plugins 2024-11-28 06:56:38,516 main DEBUG PluginManager 'Lookup' found 16 plugins 2024-11-28 06:56:38,517 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-28 06:56:38,526 main DEBUG PluginManager 'TypeConverter' found 26 plugins 2024-11-28 06:56:38,543 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.util.MBeans", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-28 06:56:38,545 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-28 06:56:38,546 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.logging.TestJul2Slf4j", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-28 06:56:38,546 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-28 06:56:38,547 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.zookeeper", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-28 06:56:38,547 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-28 06:56:38,549 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSinkAdapter", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-28 06:56:38,549 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-28 06:56:38,550 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSystemImpl", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-28 06:56:38,551 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-28 06:56:38,552 main DEBUG LoggerConfig$Builder(additivity="false", level="WARN", levelAndRefs="null", name="org.apache.directory", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-28 06:56:38,552 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-28 06:56:38,553 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.ipc.FailedServers", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-28 06:56:38,553 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-28 06:56:38,554 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsConfig", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-28 06:56:38,554 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-28 06:56:38,555 main DEBUG LoggerConfig$Builder(additivity="null", level="INFO", levelAndRefs="null", name="org.apache.hadoop.hbase.ScheduledChore", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-28 06:56:38,555 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-28 06:56:38,556 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.regionserver.RSRpcServices", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-28 06:56:38,556 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-28 06:56:38,557 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-28 06:56:38,557 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-28 06:56:38,558 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-28 06:56:38,558 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-28 06:56:38,559 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hbase.thirdparty.io.netty.channel", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-28 06:56:38,560 main DEBUG Building Plugin[name=root, class=org.apache.logging.log4j.core.config.LoggerConfig$RootLogger]. 2024-11-28 06:56:38,561 main DEBUG LoggerConfig$RootLogger$Builder(additivity="null", level="null", levelAndRefs="INFO,Console", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-28 06:56:38,563 main DEBUG Building Plugin[name=loggers, class=org.apache.logging.log4j.core.config.LoggersPlugin]. 2024-11-28 06:56:38,565 main DEBUG createLoggers(={org.apache.hadoop.metrics2.util.MBeans, org.apache.hadoop.hbase.logging.TestJul2Slf4j, org.apache.zookeeper, org.apache.hadoop.metrics2.impl.MetricsSinkAdapter, org.apache.hadoop.metrics2.impl.MetricsSystemImpl, org.apache.directory, org.apache.hadoop.hbase.ipc.FailedServers, org.apache.hadoop.metrics2.impl.MetricsConfig, org.apache.hadoop.hbase.ScheduledChore, org.apache.hadoop.hbase.regionserver.RSRpcServices, org.apache.hadoop, org.apache.hadoop.hbase, org.apache.hbase.thirdparty.io.netty.channel, root}) 2024-11-28 06:56:38,566 main DEBUG Building Plugin[name=layout, class=org.apache.logging.log4j.core.layout.PatternLayout]. 2024-11-28 06:56:38,568 main DEBUG PatternLayout$Builder(pattern="%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n", PatternSelector=null, Configuration(PropertiesConfig), Replace=null, charset="null", alwaysWriteExceptions="null", disableAnsi="null", noConsoleNoAnsi="null", header="null", footer="null") 2024-11-28 06:56:38,568 main DEBUG PluginManager 'Converter' found 47 plugins 2024-11-28 06:56:38,580 main DEBUG Building Plugin[name=appender, class=org.apache.hadoop.hbase.logging.HBaseTestAppender]. 2024-11-28 06:56:38,583 main DEBUG HBaseTestAppender$Builder(target="SYSTEM_ERR", maxSize="1G", bufferedIo="null", bufferSize="null", immediateFlush="null", ignoreExceptions="null", PatternLayout(%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n), name="Console", Configuration(PropertiesConfig), Filter=null, ={}) 2024-11-28 06:56:38,585 main DEBUG Starting HBaseTestOutputStreamManager SYSTEM_ERR 2024-11-28 06:56:38,586 main DEBUG Building Plugin[name=appenders, class=org.apache.logging.log4j.core.config.AppendersPlugin]. 2024-11-28 06:56:38,587 main DEBUG createAppenders(={Console}) 2024-11-28 06:56:38,587 main DEBUG Configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@15a34df2 initialized 2024-11-28 06:56:38,588 main DEBUG Starting configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@15a34df2 2024-11-28 06:56:38,588 main DEBUG Started configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@15a34df2 OK. 2024-11-28 06:56:38,589 main DEBUG Shutting down OutputStreamManager SYSTEM_OUT.false.false-1 2024-11-28 06:56:38,589 main DEBUG OutputStream closed 2024-11-28 06:56:38,590 main DEBUG Shut down OutputStreamManager SYSTEM_OUT.false.false-1, all resources released: true 2024-11-28 06:56:38,590 main DEBUG Appender DefaultConsole-1 stopped with status true 2024-11-28 06:56:38,590 main DEBUG Stopped org.apache.logging.log4j.core.config.DefaultConfiguration@4f2b503c OK 2024-11-28 06:56:38,694 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6 2024-11-28 06:56:38,697 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=StatusLogger 2024-11-28 06:56:38,699 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=ContextSelector 2024-11-28 06:56:38,700 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name= 2024-11-28 06:56:38,701 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.directory 2024-11-28 06:56:38,702 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSinkAdapter 2024-11-28 06:56:38,702 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.zookeeper 2024-11-28 06:56:38,703 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.logging.TestJul2Slf4j 2024-11-28 06:56:38,703 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSystemImpl 2024-11-28 06:56:38,703 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.util.MBeans 2024-11-28 06:56:38,704 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase 2024-11-28 06:56:38,704 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop 2024-11-28 06:56:38,704 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ipc.FailedServers 2024-11-28 06:56:38,705 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.regionserver.RSRpcServices 2024-11-28 06:56:38,705 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsConfig 2024-11-28 06:56:38,706 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hbase.thirdparty.io.netty.channel 2024-11-28 06:56:38,706 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ScheduledChore 2024-11-28 06:56:38,707 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Appenders,name=Console 2024-11-28 06:56:38,710 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-28 06:56:38,711 main DEBUG Reconfiguration complete for context[name=1dbd16a6] at URI jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-logging/target/hbase-logging-4.0.0-alpha-1-SNAPSHOT-tests.jar!/log4j2.properties (org.apache.logging.log4j.core.LoggerContext@31c7528f) with optional ClassLoader: null 2024-11-28 06:56:38,711 main DEBUG Shutdown hook enabled. Registering a new one. 2024-11-28 06:56:38,712 main DEBUG LoggerContext[name=1dbd16a6, org.apache.logging.log4j.core.LoggerContext@31c7528f] started OK. 2024-11-28T06:56:38,733 INFO [main {}] hbase.HBaseClassTestRule(94): Test class org.apache.hadoop.hbase.io.hfile.TestBlockEvictionOnRegionMovement timeout: 13 mins 2024-11-28 06:56:38,737 main DEBUG AsyncLogger.ThreadNameStrategy=UNCACHED (user specified null, default is UNCACHED) 2024-11-28 06:56:38,737 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-28T06:56:39,092 DEBUG [main {}] hbase.HBaseTestingUtil(323): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7e9a7384-d065-9ff8-196a-59688d3a4920 2024-11-28T06:56:39,150 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: io.hfile.TestBlockEvictionOnRegionMovement#testBlockEvictionOnRegionMove Thread=12, OpenFileDescriptor=287, MaxFileDescriptor=1048576, SystemLoadAverage=346, ProcessCount=11, AvailableMemoryMB=8483 2024-11-28T06:56:39,296 WARN [Time-limited test {}] util.NativeCodeLoader(60): Unable to load native-hadoop library for your platform... using builtin-java classes where applicable 2024-11-28T06:56:39,467 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-28T06:56:39,487 INFO [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7e9a7384-d065-9ff8-196a-59688d3a4920/cluster_1ba2156e-979b-3765-d327-70134427874d, deleteOnExit=true 2024-11-28T06:56:39,633 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7e9a7384-d065-9ff8-196a-59688d3a4920/cluster_1ba2156e-979b-3765-d327-70134427874d/zookeeper_0, clientPort=58909, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7e9a7384-d065-9ff8-196a-59688d3a4920/cluster_1ba2156e-979b-3765-d327-70134427874d/zookeeper_0/version-2, dataDirSize=457 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7e9a7384-d065-9ff8-196a-59688d3a4920/cluster_1ba2156e-979b-3765-d327-70134427874d/zookeeper_0/version-2, dataLogSize=457 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0 2024-11-28T06:56:39,647 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=58909 2024-11-28T06:56:39,649 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-28T06:56:39,767 INFO [Time-limited test {}] util.FSUtils(489): Created version file at /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7e9a7384-d065-9ff8-196a-59688d3a4920/752619e7-664e-6a55-3bd6-93cba9c9ba92 with version=8 2024-11-28T06:56:39,768 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1139): Setting hbase.fs.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7e9a7384-d065-9ff8-196a-59688d3a4920/752619e7-664e-6a55-3bd6-93cba9c9ba92/hbase-staging 2024-11-28T06:56:39,877 DEBUG [Time-limited test {}] channel.MultithreadEventLoopGroup(44): -Dio.netty.eventLoopThreads: 16 2024-11-28T06:56:40,191 INFO [Time-limited test {}] client.ConnectionUtils(128): master/1cb9cf437a36:0 server-side Connection retries=45 2024-11-28T06:56:40,202 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-28T06:56:40,208 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-28T06:56:40,236 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-28T06:56:40,236 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-28T06:56:40,237 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-28T06:56:40,446 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService 2024-11-28T06:56:40,546 WARN [Time-limited test {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-28T06:56:40,604 INFO [Time-limited test {}] metrics.MetricRegistriesLoader(60): Loaded MetricRegistries class org.apache.hadoop.hbase.metrics.impl.MetricRegistriesImpl 2024-11-28T06:56:40,618 DEBUG [Time-limited test {}] util.ClassSize(228): Using Unsafe to estimate memory layout 2024-11-28T06:56:40,623 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-28T06:56:40,668 DEBUG [Time-limited test {}] channel.DefaultChannelId(84): -Dio.netty.processId: 109185 (auto-detected) 2024-11-28T06:56:40,669 DEBUG [Time-limited test {}] channel.DefaultChannelId(106): -Dio.netty.machineId: 02:42:ac:ff:fe:11:00:02 (auto-detected) 2024-11-28T06:56:40,704 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:38499 2024-11-28T06:56:40,729 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:38499 connecting to ZooKeeper ensemble=127.0.0.1:58909 2024-11-28T06:56:40,846 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:384990x0, quorum=127.0.0.1:58909, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-28T06:56:40,849 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:38499-0x10181844d810000 connected 2024-11-28T06:56:40,933 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-28T06:56:40,934 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-28T06:56:40,943 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:38499-0x10181844d810000, quorum=127.0.0.1:58909, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-28T06:56:40,948 INFO [Time-limited test {}] master.HMaster(525): hbase.rootdir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7e9a7384-d065-9ff8-196a-59688d3a4920/752619e7-664e-6a55-3bd6-93cba9c9ba92, hbase.cluster.distributed=false 2024-11-28T06:56:40,980 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:38499-0x10181844d810000, quorum=127.0.0.1:58909, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-28T06:56:40,988 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=38499 2024-11-28T06:56:40,988 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=38499 2024-11-28T06:56:40,991 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=38499 2024-11-28T06:56:40,995 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=38499 2024-11-28T06:56:40,995 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=38499 2024-11-28T06:56:41,107 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/1cb9cf437a36:0 server-side Connection retries=45 2024-11-28T06:56:41,109 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-28T06:56:41,110 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-28T06:56:41,110 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-28T06:56:41,110 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-28T06:56:41,111 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-28T06:56:41,115 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-28T06:56:41,119 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-28T06:56:41,120 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:37673 2024-11-28T06:56:41,122 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:37673 connecting to ZooKeeper ensemble=127.0.0.1:58909 2024-11-28T06:56:41,122 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-28T06:56:41,124 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-28T06:56:41,159 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:376730x0, quorum=127.0.0.1:58909, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-28T06:56:41,160 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:37673-0x10181844d810001 connected 2024-11-28T06:56:41,160 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:37673-0x10181844d810001, quorum=127.0.0.1:58909, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-28T06:56:41,166 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-28T06:56:41,181 INFO [Time-limited test {}] bucket.FileIOEngine(99): Allocating cache 400 MB, on the path:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7e9a7384-d065-9ff8-196a-59688d3a4920/bucket.cache 2024-11-28T06:56:41,189 INFO [Time-limited test {}] bucket.BucketCache(364): Instantiating BucketCache with acceptableFactor: 0.95, minFactor: 0.85, extraFreeFactor: 0.1, singleFactor: 0.25, multiFactor: 0.5, memoryFactor: 0.25, useStrongRef: false 2024-11-28T06:56:41,191 INFO [Time-limited test {}] bucket.BucketCache(478): Starting BucketCachePersisterThread 2024-11-28T06:56:41,193 INFO [Time-limited test {}] bucket.BucketCachePersister(38): BucketCachePersister started with interval: 100 2024-11-28T06:56:41,196 INFO [Thread-10 {}] bucket.BucketCache(1530): Started retrieving bucket cache from file 2024-11-28T06:56:41,196 WARN [Thread-10 {}] bucket.BucketCache(1533): Persistence file missing! It's ok if it's first run after enabling persistent cache. 2024-11-28T06:56:41,203 INFO [Time-limited test {}] bucket.BucketCache(403): Started bucket cache; ioengine=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7e9a7384-d065-9ff8-196a-59688d3a4920/bucket.cache, capacity=400 MB, blockSize=64 KB, writerThreadNum=3, writerQLen=64, persistencePath=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7e9a7384-d065-9ff8-196a-59688d3a4920/bucket.persistence, bucketAllocator=org.apache.hadoop.hbase.io.hfile.bucket.BucketAllocator 2024-11-28T06:56:41,208 INFO [Thread-10 {}] bucket.BucketAllocator(343): Cache totalSize=418148352, buckets=199, bucket capacity=2101248=(4*525312)=(FEWEST_ITEMS_IN_BUCKET*(largest configured bucketcache size)) 2024-11-28T06:56:41,208 INFO [Thread-10 {}] bucket.BucketCache(413): Persistent bucket cache recovery from /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7e9a7384-d065-9ff8-196a-59688d3a4920/bucket.persistence is complete. 2024-11-28T06:56:41,213 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-28T06:56:41,217 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:37673-0x10181844d810001, quorum=127.0.0.1:58909, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-28T06:56:41,224 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:37673-0x10181844d810001, quorum=127.0.0.1:58909, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-28T06:56:41,234 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=37673 2024-11-28T06:56:41,235 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=37673 2024-11-28T06:56:41,239 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=37673 2024-11-28T06:56:41,243 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=37673 2024-11-28T06:56:41,243 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=37673 2024-11-28T06:56:41,265 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/1cb9cf437a36:0 server-side Connection retries=45 2024-11-28T06:56:41,266 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-28T06:56:41,266 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-28T06:56:41,266 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-28T06:56:41,267 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-28T06:56:41,267 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-28T06:56:41,267 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-28T06:56:41,268 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-28T06:56:41,270 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:35569 2024-11-28T06:56:41,273 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:35569 connecting to ZooKeeper ensemble=127.0.0.1:58909 2024-11-28T06:56:41,273 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-28T06:56:41,275 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-28T06:56:41,312 DEBUG [pool-21-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:355690x0, quorum=127.0.0.1:58909, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-28T06:56:41,314 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:355690x0, quorum=127.0.0.1:58909, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-28T06:56:41,315 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-28T06:56:41,323 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:35569-0x10181844d810002 connected 2024-11-28T06:56:41,328 INFO [Time-limited test {}] bucket.FileIOEngine(99): Allocating cache 400 MB, on the path:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7e9a7384-d065-9ff8-196a-59688d3a4920/bucket.cache 2024-11-28T06:56:41,329 INFO [Time-limited test {}] bucket.BucketCache(364): Instantiating BucketCache with acceptableFactor: 0.95, minFactor: 0.85, extraFreeFactor: 0.1, singleFactor: 0.25, multiFactor: 0.5, memoryFactor: 0.25, useStrongRef: false 2024-11-28T06:56:41,330 INFO [Time-limited test {}] bucket.BucketCache(478): Starting BucketCachePersisterThread 2024-11-28T06:56:41,330 INFO [Time-limited test {}] bucket.BucketCachePersister(38): BucketCachePersister started with interval: 100 2024-11-28T06:56:41,334 INFO [Thread-11 {}] bucket.BucketCache(1530): Started retrieving bucket cache from file 2024-11-28T06:56:41,334 WARN [Thread-11 {}] bucket.BucketCache(1533): Persistence file missing! It's ok if it's first run after enabling persistent cache. 2024-11-28T06:56:41,335 INFO [Thread-11 {}] bucket.BucketAllocator(343): Cache totalSize=418148352, buckets=199, bucket capacity=2101248=(4*525312)=(FEWEST_ITEMS_IN_BUCKET*(largest configured bucketcache size)) 2024-11-28T06:56:41,336 INFO [Thread-11 {}] bucket.BucketCache(413): Persistent bucket cache recovery from /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7e9a7384-d065-9ff8-196a-59688d3a4920/bucket.persistence is complete. 2024-11-28T06:56:41,336 INFO [Time-limited test {}] bucket.BucketCache(403): Started bucket cache; ioengine=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7e9a7384-d065-9ff8-196a-59688d3a4920/bucket.cache, capacity=400 MB, blockSize=64 KB, writerThreadNum=3, writerQLen=64, persistencePath=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7e9a7384-d065-9ff8-196a-59688d3a4920/bucket.persistence, bucketAllocator=org.apache.hadoop.hbase.io.hfile.bucket.BucketAllocator 2024-11-28T06:56:41,337 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-28T06:56:41,339 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:35569-0x10181844d810002, quorum=127.0.0.1:58909, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-28T06:56:41,343 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:35569-0x10181844d810002, quorum=127.0.0.1:58909, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-28T06:56:41,381 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=35569 2024-11-28T06:56:41,383 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=35569 2024-11-28T06:56:41,391 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=35569 2024-11-28T06:56:41,416 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=35569 2024-11-28T06:56:41,417 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=35569 2024-11-28T06:56:41,446 DEBUG [M:0;1cb9cf437a36:38499 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;1cb9cf437a36:38499 2024-11-28T06:56:41,451 INFO [master/1cb9cf437a36:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/1cb9cf437a36,38499,1732776999991 2024-11-28T06:56:41,521 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38499-0x10181844d810000, quorum=127.0.0.1:58909, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-28T06:56:41,522 DEBUG [pool-21-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35569-0x10181844d810002, quorum=127.0.0.1:58909, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-28T06:56:41,522 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37673-0x10181844d810001, quorum=127.0.0.1:58909, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-28T06:56:41,527 DEBUG [master/1cb9cf437a36:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:38499-0x10181844d810000, quorum=127.0.0.1:58909, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/1cb9cf437a36,38499,1732776999991 2024-11-28T06:56:41,570 DEBUG [pool-21-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35569-0x10181844d810002, quorum=127.0.0.1:58909, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-28T06:56:41,570 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37673-0x10181844d810001, quorum=127.0.0.1:58909, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-28T06:56:41,570 DEBUG [pool-21-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35569-0x10181844d810002, quorum=127.0.0.1:58909, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-28T06:56:41,570 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37673-0x10181844d810001, quorum=127.0.0.1:58909, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-28T06:56:41,570 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38499-0x10181844d810000, quorum=127.0.0.1:58909, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-28T06:56:41,572 DEBUG [master/1cb9cf437a36:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:38499-0x10181844d810000, quorum=127.0.0.1:58909, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-11-28T06:56:41,573 INFO [master/1cb9cf437a36:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/1cb9cf437a36,38499,1732776999991 from backup master directory 2024-11-28T06:56:41,586 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38499-0x10181844d810000, quorum=127.0.0.1:58909, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/1cb9cf437a36,38499,1732776999991 2024-11-28T06:56:41,586 DEBUG [pool-21-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35569-0x10181844d810002, quorum=127.0.0.1:58909, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-28T06:56:41,587 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37673-0x10181844d810001, quorum=127.0.0.1:58909, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-28T06:56:41,587 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38499-0x10181844d810000, quorum=127.0.0.1:58909, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-28T06:56:41,591 WARN [master/1cb9cf437a36:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-28T06:56:41,591 INFO [master/1cb9cf437a36:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=1cb9cf437a36,38499,1732776999991 2024-11-28T06:56:41,594 INFO [master/1cb9cf437a36:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating data MemStoreChunkPool with chunk size 2 MB, max count 396, initial count 0 2024-11-28T06:56:41,599 INFO [master/1cb9cf437a36:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating index MemStoreChunkPool with chunk size 204.80 KB, max count 440, initial count 0 2024-11-28T06:56:41,618 DEBUG [master/1cb9cf437a36:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7e9a7384-d065-9ff8-196a-59688d3a4920/752619e7-664e-6a55-3bd6-93cba9c9ba92/hbase.id] with ID: 4cee6445-ce15-4efc-86bd-e2c02b720f01 2024-11-28T06:56:41,618 DEBUG [master/1cb9cf437a36:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7e9a7384-d065-9ff8-196a-59688d3a4920/752619e7-664e-6a55-3bd6-93cba9c9ba92/.tmp/hbase.id 2024-11-28T06:56:41,624 DEBUG [master/1cb9cf437a36:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7e9a7384-d065-9ff8-196a-59688d3a4920/752619e7-664e-6a55-3bd6-93cba9c9ba92/.tmp/hbase.id]:[file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7e9a7384-d065-9ff8-196a-59688d3a4920/752619e7-664e-6a55-3bd6-93cba9c9ba92/hbase.id] 2024-11-28T06:56:41,679 DEBUG [master/1cb9cf437a36:0:becomeActiveMaster {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-28T06:56:41,687 INFO [master/1cb9cf437a36:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem. 2024-11-28T06:56:41,715 INFO [master/1cb9cf437a36:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 24ms. 2024-11-28T06:56:41,736 DEBUG [pool-21-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35569-0x10181844d810002, quorum=127.0.0.1:58909, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-28T06:56:41,736 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37673-0x10181844d810001, quorum=127.0.0.1:58909, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-28T06:56:41,737 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38499-0x10181844d810000, quorum=127.0.0.1:58909, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-28T06:56:41,786 INFO [master/1cb9cf437a36:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-28T06:56:41,791 INFO [master/1cb9cf437a36:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-11-28T06:56:41,854 DEBUG [master/1cb9cf437a36:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(244): No decryptEncryptedDataEncryptionKey method in DFSClient, should be hadoop version with HDFS-12396 java.lang.NoSuchMethodException: org.apache.hadoop.hdfs.DFSClient.decryptEncryptedDataEncryptionKey(org.apache.hadoop.fs.FileEncryptionInfo) at java.lang.Class.getDeclaredMethod(Class.java:2675) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelperWithoutHDFS12396(FanOutOneBlockAsyncDFSOutputSaslHelper.java:183) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelper(FanOutOneBlockAsyncDFSOutputSaslHelper.java:242) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.(FanOutOneBlockAsyncDFSOutputSaslHelper.java:253) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.lang.Class.forName0(Native Method) ~[?:?] at java.lang.Class.forName(Class.java:375) ~[?:?] at org.apache.hadoop.hbase.wal.AsyncFSWALProvider.load(AsyncFSWALProvider.java:150) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.getProviderClass(WALFactory.java:174) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:262) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:231) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegion.create(MasterRegion.java:400) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegionFactory.create(MasterRegionFactory.java:135) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.finishActiveMasterInitialization(HMaster.java:1003) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.startActiveMasterManager(HMaster.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.lambda$run$0(HMaster.java:613) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.lambda$tracedRunnable$2(TraceUtil.java:155) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T06:56:41,862 INFO [master/1cb9cf437a36:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-28T06:56:41,953 INFO [master/1cb9cf437a36:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7e9a7384-d065-9ff8-196a-59688d3a4920/752619e7-664e-6a55-3bd6-93cba9c9ba92/MasterData/data/master/store 2024-11-28T06:56:41,988 INFO [master/1cb9cf437a36:0:becomeActiveMaster {}] throttle.StoreHotnessProtector(112): StoreHotnessProtector is disabled. Set hbase.region.store.parallel.put.limit > 0 to enable, which may help mitigate load under heavy write pressure. 2024-11-28T06:56:41,993 DEBUG [master/1cb9cf437a36:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-28T06:56:41,995 DEBUG [master/1cb9cf437a36:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-28T06:56:41,995 INFO [master/1cb9cf437a36:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-28T06:56:41,995 DEBUG [master/1cb9cf437a36:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-28T06:56:41,997 DEBUG [master/1cb9cf437a36:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-28T06:56:41,998 DEBUG [master/1cb9cf437a36:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-28T06:56:41,998 INFO [master/1cb9cf437a36:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-28T06:56:41,999 DEBUG [master/1cb9cf437a36:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1732777001994Disabling compacts and flushes for region at 1732777001995 (+1 ms)Disabling writes for close at 1732777001998 (+3 ms)Writing region close event to WAL at 1732777001998Closed at 1732777001998 2024-11-28T06:56:42,000 WARN [master/1cb9cf437a36:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7e9a7384-d065-9ff8-196a-59688d3a4920/752619e7-664e-6a55-3bd6-93cba9c9ba92/MasterData/data/master/store/.initializing 2024-11-28T06:56:42,000 DEBUG [master/1cb9cf437a36:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7e9a7384-d065-9ff8-196a-59688d3a4920/752619e7-664e-6a55-3bd6-93cba9c9ba92/MasterData/WALs/1cb9cf437a36,38499,1732776999991 2024-11-28T06:56:42,028 INFO [master/1cb9cf437a36:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-28T06:56:42,044 INFO [master/1cb9cf437a36:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=1cb9cf437a36%2C38499%2C1732776999991, suffix=, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7e9a7384-d065-9ff8-196a-59688d3a4920/752619e7-664e-6a55-3bd6-93cba9c9ba92/MasterData/WALs/1cb9cf437a36,38499,1732776999991, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7e9a7384-d065-9ff8-196a-59688d3a4920/752619e7-664e-6a55-3bd6-93cba9c9ba92/MasterData/oldWALs, maxLogs=10 2024-11-28T06:56:42,094 INFO [master/1cb9cf437a36:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7e9a7384-d065-9ff8-196a-59688d3a4920/752619e7-664e-6a55-3bd6-93cba9c9ba92/MasterData/WALs/1cb9cf437a36,38499,1732776999991/1cb9cf437a36%2C38499%2C1732776999991.1732777002049 2024-11-28T06:56:42,095 DEBUG [master/1cb9cf437a36:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-11-28T06:56:42,096 DEBUG [master/1cb9cf437a36:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-11-28T06:56:42,096 DEBUG [master/1cb9cf437a36:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-28T06:56:42,101 DEBUG [master/1cb9cf437a36:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-11-28T06:56:42,103 DEBUG [master/1cb9cf437a36:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-11-28T06:56:42,154 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-11-28T06:56:42,214 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-11-28T06:56:42,231 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:56:42,233 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-28T06:56:42,234 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-11-28T06:56:42,256 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-11-28T06:56:42,257 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:56:42,257 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-28T06:56:42,258 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-11-28T06:56:42,274 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-11-28T06:56:42,274 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:56:42,277 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-28T06:56:42,280 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-11-28T06:56:42,324 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-11-28T06:56:42,324 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:56:42,325 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-28T06:56:42,326 DEBUG [master/1cb9cf437a36:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682 2024-11-28T06:56:42,330 DEBUG [master/1cb9cf437a36:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7e9a7384-d065-9ff8-196a-59688d3a4920/752619e7-664e-6a55-3bd6-93cba9c9ba92/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-28T06:56:42,331 DEBUG [master/1cb9cf437a36:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7e9a7384-d065-9ff8-196a-59688d3a4920/752619e7-664e-6a55-3bd6-93cba9c9ba92/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-28T06:56:42,335 DEBUG [master/1cb9cf437a36:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682 2024-11-28T06:56:42,336 DEBUG [master/1cb9cf437a36:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682 2024-11-28T06:56:42,340 DEBUG [master/1cb9cf437a36:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-28T06:56:42,349 DEBUG [master/1cb9cf437a36:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-11-28T06:56:42,401 DEBUG [master/1cb9cf437a36:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7e9a7384-d065-9ff8-196a-59688d3a4920/752619e7-664e-6a55-3bd6-93cba9c9ba92/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-28T06:56:42,402 INFO [master/1cb9cf437a36:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=59909385, jitterRate=-0.10728059709072113}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-28T06:56:42,420 DEBUG [master/1cb9cf437a36:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682: Writing region info on filesystem at 1732777002120Initializing all the Stores at 1732777002121 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732777002121Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732777002123 (+2 ms)Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732777002124 (+1 ms)Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732777002124Cleaning up temporary data from old regions at 1732777002336 (+212 ms)Region opened successfully at 1732777002420 (+84 ms) 2024-11-28T06:56:42,432 INFO [master/1cb9cf437a36:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-11-28T06:56:42,540 DEBUG [master/1cb9cf437a36:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6c82f2fa, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=1cb9cf437a36/172.17.0.2:0 2024-11-28T06:56:42,590 INFO [master/1cb9cf437a36:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating... 2024-11-28T06:56:42,609 INFO [master/1cb9cf437a36:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5 2024-11-28T06:56:42,610 INFO [master/1cb9cf437a36:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50 2024-11-28T06:56:42,615 INFO [master/1cb9cf437a36:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-28T06:56:42,617 INFO [master/1cb9cf437a36:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 1 msec 2024-11-28T06:56:42,639 INFO [master/1cb9cf437a36:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 21 msec 2024-11-28T06:56:42,639 INFO [master/1cb9cf437a36:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-11-28T06:56:42,683 INFO [master/1cb9cf437a36:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2024-11-28T06:56:42,696 DEBUG [master/1cb9cf437a36:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:38499-0x10181844d810000, quorum=127.0.0.1:58909, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error) 2024-11-28T06:56:42,742 DEBUG [master/1cb9cf437a36:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false 2024-11-28T06:56:42,753 INFO [master/1cb9cf437a36:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2024-11-28T06:56:42,757 DEBUG [master/1cb9cf437a36:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:38499-0x10181844d810000, quorum=127.0.0.1:58909, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error) 2024-11-28T06:56:42,769 DEBUG [master/1cb9cf437a36:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false 2024-11-28T06:56:42,772 INFO [master/1cb9cf437a36:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2024-11-28T06:56:42,776 DEBUG [master/1cb9cf437a36:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:38499-0x10181844d810000, quorum=127.0.0.1:58909, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error) 2024-11-28T06:56:42,786 DEBUG [master/1cb9cf437a36:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false 2024-11-28T06:56:42,788 DEBUG [master/1cb9cf437a36:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:38499-0x10181844d810000, quorum=127.0.0.1:58909, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error) 2024-11-28T06:56:42,799 DEBUG [master/1cb9cf437a36:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false 2024-11-28T06:56:42,818 DEBUG [master/1cb9cf437a36:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:38499-0x10181844d810000, quorum=127.0.0.1:58909, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error) 2024-11-28T06:56:42,828 DEBUG [master/1cb9cf437a36:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false 2024-11-28T06:56:42,853 DEBUG [pool-21-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35569-0x10181844d810002, quorum=127.0.0.1:58909, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-28T06:56:42,853 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37673-0x10181844d810001, quorum=127.0.0.1:58909, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-28T06:56:42,853 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37673-0x10181844d810001, quorum=127.0.0.1:58909, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-28T06:56:42,854 DEBUG [pool-21-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35569-0x10181844d810002, quorum=127.0.0.1:58909, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-28T06:56:42,854 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38499-0x10181844d810000, quorum=127.0.0.1:58909, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-28T06:56:42,854 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38499-0x10181844d810000, quorum=127.0.0.1:58909, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-28T06:56:42,859 INFO [master/1cb9cf437a36:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=1cb9cf437a36,38499,1732776999991, sessionid=0x10181844d810000, setting cluster-up flag (Was=false) 2024-11-28T06:56:42,886 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38499-0x10181844d810000, quorum=127.0.0.1:58909, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-28T06:56:42,886 DEBUG [pool-21-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35569-0x10181844d810002, quorum=127.0.0.1:58909, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-28T06:56:42,886 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37673-0x10181844d810001, quorum=127.0.0.1:58909, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-28T06:56:42,911 DEBUG [master/1cb9cf437a36:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort 2024-11-28T06:56:42,913 DEBUG [master/1cb9cf437a36:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=1cb9cf437a36,38499,1732776999991 2024-11-28T06:56:42,936 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38499-0x10181844d810000, quorum=127.0.0.1:58909, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-28T06:56:42,936 DEBUG [pool-21-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35569-0x10181844d810002, quorum=127.0.0.1:58909, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-28T06:56:42,936 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37673-0x10181844d810001, quorum=127.0.0.1:58909, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-28T06:56:42,961 DEBUG [master/1cb9cf437a36:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort 2024-11-28T06:56:42,964 DEBUG [master/1cb9cf437a36:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=1cb9cf437a36,38499,1732776999991 2024-11-28T06:56:42,985 INFO [master/1cb9cf437a36:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7e9a7384-d065-9ff8-196a-59688d3a4920/752619e7-664e-6a55-3bd6-93cba9c9ba92/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again 2024-11-28T06:56:43,029 INFO [RS:0;1cb9cf437a36:37673 {}] regionserver.HRegionServer(746): ClusterId : 4cee6445-ce15-4efc-86bd-e2c02b720f01 2024-11-28T06:56:43,032 DEBUG [RS:0;1cb9cf437a36:37673 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-28T06:56:43,034 INFO [RS:1;1cb9cf437a36:35569 {}] regionserver.HRegionServer(746): ClusterId : 4cee6445-ce15-4efc-86bd-e2c02b720f01 2024-11-28T06:56:43,034 DEBUG [RS:1;1cb9cf437a36:35569 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-28T06:56:43,064 DEBUG [RS:0;1cb9cf437a36:37673 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-28T06:56:43,064 DEBUG [RS:1;1cb9cf437a36:35569 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-28T06:56:43,064 DEBUG [RS:0;1cb9cf437a36:37673 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-28T06:56:43,064 DEBUG [RS:1;1cb9cf437a36:35569 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-28T06:56:43,078 DEBUG [RS:1;1cb9cf437a36:35569 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-28T06:56:43,078 DEBUG [RS:0;1cb9cf437a36:37673 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-28T06:56:43,079 DEBUG [RS:1;1cb9cf437a36:35569 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@13d34603, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=1cb9cf437a36/172.17.0.2:0 2024-11-28T06:56:43,079 DEBUG [RS:0;1cb9cf437a36:37673 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@79299e97, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=1cb9cf437a36/172.17.0.2:0 2024-11-28T06:56:43,089 DEBUG [master/1cb9cf437a36:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-28T06:56:43,101 DEBUG [RS:0;1cb9cf437a36:37673 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;1cb9cf437a36:37673 2024-11-28T06:56:43,102 DEBUG [RS:1;1cb9cf437a36:35569 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:1;1cb9cf437a36:35569 2024-11-28T06:56:43,102 INFO [master/1cb9cf437a36:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2 2024-11-28T06:56:43,107 INFO [RS:0;1cb9cf437a36:37673 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-28T06:56:43,107 INFO [RS:1;1cb9cf437a36:35569 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-28T06:56:43,107 INFO [RS:1;1cb9cf437a36:35569 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-28T06:56:43,107 INFO [RS:0;1cb9cf437a36:37673 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-28T06:56:43,107 DEBUG [RS:1;1cb9cf437a36:35569 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-28T06:56:43,107 DEBUG [RS:0;1cb9cf437a36:37673 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-28T06:56:43,111 INFO [RS:1;1cb9cf437a36:35569 {}] regionserver.HRegionServer(2659): reportForDuty to master=1cb9cf437a36,38499,1732776999991 with port=35569, startcode=1732777001264 2024-11-28T06:56:43,113 INFO [RS:0;1cb9cf437a36:37673 {}] regionserver.HRegionServer(2659): reportForDuty to master=1cb9cf437a36,38499,1732776999991 with port=37673, startcode=1732777001059 2024-11-28T06:56:43,113 INFO [master/1cb9cf437a36:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2024-11-28T06:56:43,125 DEBUG [master/1cb9cf437a36:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: 1cb9cf437a36,38499,1732776999991 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2024-11-28T06:56:43,132 DEBUG [RS:0;1cb9cf437a36:37673 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-28T06:56:43,134 DEBUG [RS:1;1cb9cf437a36:35569 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-28T06:56:43,136 DEBUG [master/1cb9cf437a36:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/1cb9cf437a36:0, corePoolSize=5, maxPoolSize=5 2024-11-28T06:56:43,137 DEBUG [master/1cb9cf437a36:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/1cb9cf437a36:0, corePoolSize=5, maxPoolSize=5 2024-11-28T06:56:43,137 DEBUG [master/1cb9cf437a36:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/1cb9cf437a36:0, corePoolSize=5, maxPoolSize=5 2024-11-28T06:56:43,137 DEBUG [master/1cb9cf437a36:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/1cb9cf437a36:0, corePoolSize=5, maxPoolSize=5 2024-11-28T06:56:43,138 DEBUG [master/1cb9cf437a36:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/1cb9cf437a36:0, corePoolSize=10, maxPoolSize=10 2024-11-28T06:56:43,138 DEBUG [master/1cb9cf437a36:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/1cb9cf437a36:0, corePoolSize=1, maxPoolSize=1 2024-11-28T06:56:43,139 DEBUG [master/1cb9cf437a36:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/1cb9cf437a36:0, corePoolSize=2, maxPoolSize=2 2024-11-28T06:56:43,139 DEBUG [master/1cb9cf437a36:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/1cb9cf437a36:0, corePoolSize=1, maxPoolSize=1 2024-11-28T06:56:43,145 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-28T06:56:43,146 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region 2024-11-28T06:56:43,152 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:56:43,153 INFO [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-11-28T06:56:43,187 INFO [master/1cb9cf437a36:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732777033187 2024-11-28T06:56:43,189 INFO [master/1cb9cf437a36:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2024-11-28T06:56:43,190 INFO [master/1cb9cf437a36:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2024-11-28T06:56:43,193 INFO [master/1cb9cf437a36:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2024-11-28T06:56:43,193 INFO [master/1cb9cf437a36:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2024-11-28T06:56:43,193 INFO [master/1cb9cf437a36:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2024-11-28T06:56:43,193 INFO [master/1cb9cf437a36:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2024-11-28T06:56:43,202 INFO [master/1cb9cf437a36:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-28T06:56:43,211 INFO [master/1cb9cf437a36:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2024-11-28T06:56:43,213 INFO [master/1cb9cf437a36:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2024-11-28T06:56:43,214 INFO [master/1cb9cf437a36:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2024-11-28T06:56:43,219 INFO [master/1cb9cf437a36:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2024-11-28T06:56:43,226 INFO [master/1cb9cf437a36:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2024-11-28T06:56:43,236 DEBUG [master/1cb9cf437a36:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/1cb9cf437a36:0:becomeActiveMaster-HFileCleaner.large.0-1732777003228,5,FailOnTimeoutGroup] 2024-11-28T06:56:43,238 DEBUG [master/1cb9cf437a36:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/1cb9cf437a36:0:becomeActiveMaster-HFileCleaner.small.0-1732777003236,5,FailOnTimeoutGroup] 2024-11-28T06:56:43,238 INFO [master/1cb9cf437a36:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-28T06:56:43,238 INFO [master/1cb9cf437a36:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2024-11-28T06:56:43,239 INFO [master/1cb9cf437a36:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2024-11-28T06:56:43,240 INFO [master/1cb9cf437a36:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2024-11-28T06:56:43,254 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44135, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=RegionServerStatusService 2024-11-28T06:56:43,254 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50853, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=RegionServerStatusService 2024-11-28T06:56:43,255 INFO [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7e9a7384-d065-9ff8-196a-59688d3a4920/752619e7-664e-6a55-3bd6-93cba9c9ba92/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321 2024-11-28T06:56:43,256 INFO [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7e9a7384-d065-9ff8-196a-59688d3a4920/752619e7-664e-6a55-3bd6-93cba9c9ba92 2024-11-28T06:56:43,265 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=38499 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 1cb9cf437a36,37673,1732777001059 2024-11-28T06:56:43,268 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=38499 {}] master.ServerManager(517): Registering regionserver=1cb9cf437a36,37673,1732777001059 2024-11-28T06:56:43,283 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=38499 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 1cb9cf437a36,35569,1732777001264 2024-11-28T06:56:43,283 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=38499 {}] master.ServerManager(517): Registering regionserver=1cb9cf437a36,35569,1732777001264 2024-11-28T06:56:43,287 DEBUG [RS:0;1cb9cf437a36:37673 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7e9a7384-d065-9ff8-196a-59688d3a4920/752619e7-664e-6a55-3bd6-93cba9c9ba92 2024-11-28T06:56:43,287 DEBUG [RS:0;1cb9cf437a36:37673 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=file:/ 2024-11-28T06:56:43,287 DEBUG [RS:0;1cb9cf437a36:37673 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-28T06:56:43,292 DEBUG [RS:1;1cb9cf437a36:35569 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7e9a7384-d065-9ff8-196a-59688d3a4920/752619e7-664e-6a55-3bd6-93cba9c9ba92 2024-11-28T06:56:43,292 DEBUG [RS:1;1cb9cf437a36:35569 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=file:/ 2024-11-28T06:56:43,292 DEBUG [RS:1;1cb9cf437a36:35569 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-28T06:56:43,304 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-28T06:56:43,315 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family info of region 1588230740 2024-11-28T06:56:43,319 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38499-0x10181844d810000, quorum=127.0.0.1:58909, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-28T06:56:43,328 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-28T06:56:43,328 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:56:43,329 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-28T06:56:43,329 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family ns of region 1588230740 2024-11-28T06:56:43,334 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-28T06:56:43,334 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:56:43,334 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-28T06:56:43,335 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family rep_barrier of region 1588230740 2024-11-28T06:56:43,340 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-28T06:56:43,340 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:56:43,340 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-28T06:56:43,341 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family table of region 1588230740 2024-11-28T06:56:43,347 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-28T06:56:43,348 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:56:43,348 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-28T06:56:43,349 DEBUG [RS:0;1cb9cf437a36:37673 {}] zookeeper.ZKUtil(111): regionserver:37673-0x10181844d810001, quorum=127.0.0.1:58909, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/1cb9cf437a36,37673,1732777001059 2024-11-28T06:56:43,349 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-28T06:56:43,349 WARN [RS:0;1cb9cf437a36:37673 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-28T06:56:43,349 INFO [RS:0;1cb9cf437a36:37673 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-28T06:56:43,350 DEBUG [RS:0;1cb9cf437a36:37673 {}] regionserver.HRegionServer(1793): logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7e9a7384-d065-9ff8-196a-59688d3a4920/752619e7-664e-6a55-3bd6-93cba9c9ba92/WALs/1cb9cf437a36,37673,1732777001059 2024-11-28T06:56:43,350 DEBUG [RS:1;1cb9cf437a36:35569 {}] zookeeper.ZKUtil(111): regionserver:35569-0x10181844d810002, quorum=127.0.0.1:58909, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/1cb9cf437a36,35569,1732777001264 2024-11-28T06:56:43,350 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7e9a7384-d065-9ff8-196a-59688d3a4920/752619e7-664e-6a55-3bd6-93cba9c9ba92/data/hbase/meta/1588230740 2024-11-28T06:56:43,350 WARN [RS:1;1cb9cf437a36:35569 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-28T06:56:43,351 INFO [RS:1;1cb9cf437a36:35569 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-28T06:56:43,351 DEBUG [RS:1;1cb9cf437a36:35569 {}] regionserver.HRegionServer(1793): logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7e9a7384-d065-9ff8-196a-59688d3a4920/752619e7-664e-6a55-3bd6-93cba9c9ba92/WALs/1cb9cf437a36,35569,1732777001264 2024-11-28T06:56:43,351 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7e9a7384-d065-9ff8-196a-59688d3a4920/752619e7-664e-6a55-3bd6-93cba9c9ba92/data/hbase/meta/1588230740 2024-11-28T06:56:43,351 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [1cb9cf437a36,37673,1732777001059] 2024-11-28T06:56:43,351 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [1cb9cf437a36,35569,1732777001264] 2024-11-28T06:56:43,352 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-28T06:56:43,352 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-28T06:56:43,353 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-28T06:56:43,354 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-28T06:56:43,361 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7e9a7384-d065-9ff8-196a-59688d3a4920/752619e7-664e-6a55-3bd6-93cba9c9ba92/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-28T06:56:43,362 INFO [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=73114009, jitterRate=0.08948363363742828}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-28T06:56:43,367 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740: Writing region info on filesystem at 1732777003305Initializing all the Stores at 1732777003305Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732777003305Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732777003315 (+10 ms)Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732777003315Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732777003315Cleaning up temporary data from old regions at 1732777003352 (+37 ms)Region opened successfully at 1732777003366 (+14 ms) 2024-11-28T06:56:43,367 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-28T06:56:43,367 INFO [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-28T06:56:43,367 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-28T06:56:43,368 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-28T06:56:43,368 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-28T06:56:43,375 INFO [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-28T06:56:43,376 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1732777003367Disabling compacts and flushes for region at 1732777003367Disabling writes for close at 1732777003368 (+1 ms)Writing region close event to WAL at 1732777003375 (+7 ms)Closed at 1732777003375 2024-11-28T06:56:43,378 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-28T06:56:43,378 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta 2024-11-28T06:56:43,390 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-11-28T06:56:43,402 INFO [RS:0;1cb9cf437a36:37673 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-28T06:56:43,403 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-28T06:56:43,405 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2024-11-28T06:56:43,407 INFO [RS:1;1cb9cf437a36:35569 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-28T06:56:43,428 INFO [RS:0;1cb9cf437a36:37673 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-28T06:56:43,435 INFO [RS:0;1cb9cf437a36:37673 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-28T06:56:43,435 INFO [RS:0;1cb9cf437a36:37673 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-28T06:56:43,437 INFO [RS:1;1cb9cf437a36:35569 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-28T06:56:43,439 INFO [RS:0;1cb9cf437a36:37673 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-28T06:56:43,441 INFO [RS:1;1cb9cf437a36:35569 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-28T06:56:43,441 INFO [RS:1;1cb9cf437a36:35569 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-28T06:56:43,441 INFO [RS:1;1cb9cf437a36:35569 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-28T06:56:43,445 INFO [RS:0;1cb9cf437a36:37673 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-28T06:56:43,446 INFO [RS:0;1cb9cf437a36:37673 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-28T06:56:43,446 DEBUG [RS:0;1cb9cf437a36:37673 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/1cb9cf437a36:0, corePoolSize=1, maxPoolSize=1 2024-11-28T06:56:43,446 DEBUG [RS:0;1cb9cf437a36:37673 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/1cb9cf437a36:0, corePoolSize=1, maxPoolSize=1 2024-11-28T06:56:43,447 DEBUG [RS:0;1cb9cf437a36:37673 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/1cb9cf437a36:0, corePoolSize=1, maxPoolSize=1 2024-11-28T06:56:43,447 DEBUG [RS:0;1cb9cf437a36:37673 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/1cb9cf437a36:0, corePoolSize=1, maxPoolSize=1 2024-11-28T06:56:43,447 DEBUG [RS:0;1cb9cf437a36:37673 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/1cb9cf437a36:0, corePoolSize=1, maxPoolSize=1 2024-11-28T06:56:43,447 DEBUG [RS:0;1cb9cf437a36:37673 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/1cb9cf437a36:0, corePoolSize=2, maxPoolSize=2 2024-11-28T06:56:43,447 DEBUG [RS:0;1cb9cf437a36:37673 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/1cb9cf437a36:0, corePoolSize=1, maxPoolSize=1 2024-11-28T06:56:43,447 DEBUG [RS:0;1cb9cf437a36:37673 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/1cb9cf437a36:0, corePoolSize=1, maxPoolSize=1 2024-11-28T06:56:43,447 DEBUG [RS:0;1cb9cf437a36:37673 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/1cb9cf437a36:0, corePoolSize=1, maxPoolSize=1 2024-11-28T06:56:43,448 DEBUG [RS:0;1cb9cf437a36:37673 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/1cb9cf437a36:0, corePoolSize=1, maxPoolSize=1 2024-11-28T06:56:43,448 DEBUG [RS:0;1cb9cf437a36:37673 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/1cb9cf437a36:0, corePoolSize=1, maxPoolSize=1 2024-11-28T06:56:43,448 DEBUG [RS:0;1cb9cf437a36:37673 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/1cb9cf437a36:0, corePoolSize=1, maxPoolSize=1 2024-11-28T06:56:43,448 DEBUG [RS:0;1cb9cf437a36:37673 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/1cb9cf437a36:0, corePoolSize=3, maxPoolSize=3 2024-11-28T06:56:43,448 DEBUG [RS:0;1cb9cf437a36:37673 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0, corePoolSize=3, maxPoolSize=3 2024-11-28T06:56:43,449 INFO [RS:1;1cb9cf437a36:35569 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-28T06:56:43,450 INFO [RS:1;1cb9cf437a36:35569 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-28T06:56:43,450 DEBUG [RS:1;1cb9cf437a36:35569 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/1cb9cf437a36:0, corePoolSize=1, maxPoolSize=1 2024-11-28T06:56:43,450 DEBUG [RS:1;1cb9cf437a36:35569 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/1cb9cf437a36:0, corePoolSize=1, maxPoolSize=1 2024-11-28T06:56:43,450 DEBUG [RS:1;1cb9cf437a36:35569 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/1cb9cf437a36:0, corePoolSize=1, maxPoolSize=1 2024-11-28T06:56:43,451 DEBUG [RS:1;1cb9cf437a36:35569 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/1cb9cf437a36:0, corePoolSize=1, maxPoolSize=1 2024-11-28T06:56:43,451 DEBUG [RS:1;1cb9cf437a36:35569 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/1cb9cf437a36:0, corePoolSize=1, maxPoolSize=1 2024-11-28T06:56:43,451 DEBUG [RS:1;1cb9cf437a36:35569 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/1cb9cf437a36:0, corePoolSize=2, maxPoolSize=2 2024-11-28T06:56:43,451 DEBUG [RS:1;1cb9cf437a36:35569 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/1cb9cf437a36:0, corePoolSize=1, maxPoolSize=1 2024-11-28T06:56:43,451 DEBUG [RS:1;1cb9cf437a36:35569 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/1cb9cf437a36:0, corePoolSize=1, maxPoolSize=1 2024-11-28T06:56:43,452 DEBUG [RS:1;1cb9cf437a36:35569 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/1cb9cf437a36:0, corePoolSize=1, maxPoolSize=1 2024-11-28T06:56:43,452 DEBUG [RS:1;1cb9cf437a36:35569 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/1cb9cf437a36:0, corePoolSize=1, maxPoolSize=1 2024-11-28T06:56:43,452 DEBUG [RS:1;1cb9cf437a36:35569 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/1cb9cf437a36:0, corePoolSize=1, maxPoolSize=1 2024-11-28T06:56:43,452 DEBUG [RS:1;1cb9cf437a36:35569 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/1cb9cf437a36:0, corePoolSize=1, maxPoolSize=1 2024-11-28T06:56:43,452 DEBUG [RS:1;1cb9cf437a36:35569 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/1cb9cf437a36:0, corePoolSize=3, maxPoolSize=3 2024-11-28T06:56:43,453 DEBUG [RS:1;1cb9cf437a36:35569 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0, corePoolSize=3, maxPoolSize=3 2024-11-28T06:56:43,460 INFO [RS:0;1cb9cf437a36:37673 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-28T06:56:43,460 INFO [RS:0;1cb9cf437a36:37673 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-28T06:56:43,460 INFO [RS:0;1cb9cf437a36:37673 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-28T06:56:43,460 INFO [RS:0;1cb9cf437a36:37673 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-28T06:56:43,460 INFO [RS:0;1cb9cf437a36:37673 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-28T06:56:43,460 INFO [RS:0;1cb9cf437a36:37673 {}] hbase.ChoreService(168): Chore ScheduledChore name=1cb9cf437a36,37673,1732777001059-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-28T06:56:43,467 INFO [RS:1;1cb9cf437a36:35569 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-28T06:56:43,468 INFO [RS:1;1cb9cf437a36:35569 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-28T06:56:43,468 INFO [RS:1;1cb9cf437a36:35569 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-28T06:56:43,468 INFO [RS:1;1cb9cf437a36:35569 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-28T06:56:43,468 INFO [RS:1;1cb9cf437a36:35569 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-28T06:56:43,468 INFO [RS:1;1cb9cf437a36:35569 {}] hbase.ChoreService(168): Chore ScheduledChore name=1cb9cf437a36,35569,1732777001264-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-28T06:56:43,498 INFO [RS:0;1cb9cf437a36:37673 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-28T06:56:43,499 INFO [RS:1;1cb9cf437a36:35569 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-28T06:56:43,501 INFO [RS:0;1cb9cf437a36:37673 {}] hbase.ChoreService(168): Chore ScheduledChore name=1cb9cf437a36,37673,1732777001059-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-28T06:56:43,502 INFO [RS:0;1cb9cf437a36:37673 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-28T06:56:43,503 INFO [RS:0;1cb9cf437a36:37673 {}] regionserver.Replication(171): 1cb9cf437a36,37673,1732777001059 started 2024-11-28T06:56:43,510 INFO [RS:1;1cb9cf437a36:35569 {}] hbase.ChoreService(168): Chore ScheduledChore name=1cb9cf437a36,35569,1732777001264-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-28T06:56:43,511 INFO [RS:1;1cb9cf437a36:35569 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-28T06:56:43,511 INFO [RS:1;1cb9cf437a36:35569 {}] regionserver.Replication(171): 1cb9cf437a36,35569,1732777001264 started 2024-11-28T06:56:43,537 INFO [RS:1;1cb9cf437a36:35569 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-28T06:56:43,537 INFO [RS:0;1cb9cf437a36:37673 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-28T06:56:43,538 INFO [RS:1;1cb9cf437a36:35569 {}] regionserver.HRegionServer(1482): Serving as 1cb9cf437a36,35569,1732777001264, RpcServer on 1cb9cf437a36/172.17.0.2:35569, sessionid=0x10181844d810002 2024-11-28T06:56:43,538 INFO [RS:0;1cb9cf437a36:37673 {}] regionserver.HRegionServer(1482): Serving as 1cb9cf437a36,37673,1732777001059, RpcServer on 1cb9cf437a36/172.17.0.2:37673, sessionid=0x10181844d810001 2024-11-28T06:56:43,539 DEBUG [RS:1;1cb9cf437a36:35569 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-28T06:56:43,539 DEBUG [RS:1;1cb9cf437a36:35569 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 1cb9cf437a36,35569,1732777001264 2024-11-28T06:56:43,539 DEBUG [RS:1;1cb9cf437a36:35569 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '1cb9cf437a36,35569,1732777001264' 2024-11-28T06:56:43,540 DEBUG [RS:1;1cb9cf437a36:35569 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-28T06:56:43,540 DEBUG [RS:0;1cb9cf437a36:37673 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-28T06:56:43,540 DEBUG [RS:0;1cb9cf437a36:37673 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 1cb9cf437a36,37673,1732777001059 2024-11-28T06:56:43,540 DEBUG [RS:0;1cb9cf437a36:37673 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '1cb9cf437a36,37673,1732777001059' 2024-11-28T06:56:43,541 DEBUG [RS:0;1cb9cf437a36:37673 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-28T06:56:43,542 DEBUG [RS:0;1cb9cf437a36:37673 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-28T06:56:43,542 DEBUG [RS:1;1cb9cf437a36:35569 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-28T06:56:43,543 DEBUG [RS:0;1cb9cf437a36:37673 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-28T06:56:43,543 DEBUG [RS:0;1cb9cf437a36:37673 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-28T06:56:43,543 DEBUG [RS:1;1cb9cf437a36:35569 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-28T06:56:43,543 DEBUG [RS:0;1cb9cf437a36:37673 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 1cb9cf437a36,37673,1732777001059 2024-11-28T06:56:43,543 DEBUG [RS:1;1cb9cf437a36:35569 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-28T06:56:43,543 DEBUG [RS:0;1cb9cf437a36:37673 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '1cb9cf437a36,37673,1732777001059' 2024-11-28T06:56:43,543 DEBUG [RS:1;1cb9cf437a36:35569 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 1cb9cf437a36,35569,1732777001264 2024-11-28T06:56:43,543 DEBUG [RS:0;1cb9cf437a36:37673 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-28T06:56:43,543 DEBUG [RS:1;1cb9cf437a36:35569 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '1cb9cf437a36,35569,1732777001264' 2024-11-28T06:56:43,543 DEBUG [RS:1;1cb9cf437a36:35569 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-28T06:56:43,545 DEBUG [RS:1;1cb9cf437a36:35569 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-28T06:56:43,545 DEBUG [RS:0;1cb9cf437a36:37673 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-28T06:56:43,546 DEBUG [RS:1;1cb9cf437a36:35569 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-28T06:56:43,546 INFO [RS:1;1cb9cf437a36:35569 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-28T06:56:43,546 INFO [RS:1;1cb9cf437a36:35569 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-28T06:56:43,546 DEBUG [RS:0;1cb9cf437a36:37673 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-28T06:56:43,546 INFO [RS:0;1cb9cf437a36:37673 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-28T06:56:43,547 INFO [RS:0;1cb9cf437a36:37673 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-28T06:56:43,556 WARN [1cb9cf437a36:38499 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-28T06:56:43,651 INFO [RS:1;1cb9cf437a36:35569 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-28T06:56:43,651 INFO [RS:0;1cb9cf437a36:37673 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-28T06:56:43,652 INFO [RS:1;1cb9cf437a36:35569 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=1cb9cf437a36%2C35569%2C1732777001264, suffix=, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7e9a7384-d065-9ff8-196a-59688d3a4920/752619e7-664e-6a55-3bd6-93cba9c9ba92/WALs/1cb9cf437a36,35569,1732777001264, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7e9a7384-d065-9ff8-196a-59688d3a4920/752619e7-664e-6a55-3bd6-93cba9c9ba92/oldWALs, maxLogs=55 2024-11-28T06:56:43,652 INFO [RS:0;1cb9cf437a36:37673 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=1cb9cf437a36%2C37673%2C1732777001059, suffix=, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7e9a7384-d065-9ff8-196a-59688d3a4920/752619e7-664e-6a55-3bd6-93cba9c9ba92/WALs/1cb9cf437a36,37673,1732777001059, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7e9a7384-d065-9ff8-196a-59688d3a4920/752619e7-664e-6a55-3bd6-93cba9c9ba92/oldWALs, maxLogs=55 2024-11-28T06:56:43,674 INFO [RS:1;1cb9cf437a36:35569 {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7e9a7384-d065-9ff8-196a-59688d3a4920/752619e7-664e-6a55-3bd6-93cba9c9ba92/WALs/1cb9cf437a36,35569,1732777001264/1cb9cf437a36%2C35569%2C1732777001264.1732777003656 2024-11-28T06:56:43,675 DEBUG [RS:1;1cb9cf437a36:35569 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-11-28T06:56:43,692 INFO [RS:0;1cb9cf437a36:37673 {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7e9a7384-d065-9ff8-196a-59688d3a4920/752619e7-664e-6a55-3bd6-93cba9c9ba92/WALs/1cb9cf437a36,37673,1732777001059/1cb9cf437a36%2C37673%2C1732777001059.1732777003656 2024-11-28T06:56:43,693 DEBUG [RS:0;1cb9cf437a36:37673 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-11-28T06:56:43,810 DEBUG [1cb9cf437a36:38499 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=2, allServersCount=2 2024-11-28T06:56:43,823 DEBUG [1cb9cf437a36:38499 {}] balancer.BalancerClusterState(204): Hosts are {1cb9cf437a36=0} racks are {/default-rack=0} 2024-11-28T06:56:43,827 DEBUG [1cb9cf437a36:38499 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-11-28T06:56:43,827 DEBUG [1cb9cf437a36:38499 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-11-28T06:56:43,827 DEBUG [1cb9cf437a36:38499 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-11-28T06:56:43,827 DEBUG [1cb9cf437a36:38499 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-11-28T06:56:43,827 INFO [1cb9cf437a36:38499 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-11-28T06:56:43,827 INFO [1cb9cf437a36:38499 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-11-28T06:56:43,827 DEBUG [1cb9cf437a36:38499 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-11-28T06:56:43,836 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=1cb9cf437a36,35569,1732777001264 2024-11-28T06:56:43,841 INFO [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 1cb9cf437a36,35569,1732777001264, state=OPENING 2024-11-28T06:56:43,894 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2024-11-28T06:56:43,902 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38499-0x10181844d810000, quorum=127.0.0.1:58909, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-28T06:56:43,902 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37673-0x10181844d810001, quorum=127.0.0.1:58909, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-28T06:56:43,903 DEBUG [pool-21-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35569-0x10181844d810002, quorum=127.0.0.1:58909, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-28T06:56:43,904 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-28T06:56:43,904 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-28T06:56:43,904 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-28T06:56:43,907 DEBUG [PEWorker-3 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-28T06:56:43,910 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=1cb9cf437a36,35569,1732777001264}] 2024-11-28T06:56:44,084 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-28T06:56:44,087 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-4-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41913, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-28T06:56:44,097 INFO [RS_OPEN_META-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-11-28T06:56:44,097 INFO [RS_OPEN_META-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-28T06:56:44,098 INFO [RS_OPEN_META-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-11-28T06:56:44,100 INFO [RS_OPEN_META-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=1cb9cf437a36%2C35569%2C1732777001264.meta, suffix=.meta, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7e9a7384-d065-9ff8-196a-59688d3a4920/752619e7-664e-6a55-3bd6-93cba9c9ba92/WALs/1cb9cf437a36,35569,1732777001264, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7e9a7384-d065-9ff8-196a-59688d3a4920/752619e7-664e-6a55-3bd6-93cba9c9ba92/oldWALs, maxLogs=55 2024-11-28T06:56:44,128 INFO [RS_OPEN_META-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7e9a7384-d065-9ff8-196a-59688d3a4920/752619e7-664e-6a55-3bd6-93cba9c9ba92/WALs/1cb9cf437a36,35569,1732777001264/1cb9cf437a36%2C35569%2C1732777001264.meta.1732777004103.meta 2024-11-28T06:56:44,128 DEBUG [RS_OPEN_META-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-11-28T06:56:44,129 DEBUG [RS_OPEN_META-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-11-28T06:56:44,131 DEBUG [RS_OPEN_META-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-11-28T06:56:44,134 DEBUG [RS_OPEN_META-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-11-28T06:56:44,140 INFO [RS_OPEN_META-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-11-28T06:56:44,145 DEBUG [RS_OPEN_META-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-11-28T06:56:44,146 DEBUG [RS_OPEN_META-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-28T06:56:44,146 DEBUG [RS_OPEN_META-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-11-28T06:56:44,147 DEBUG [RS_OPEN_META-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-11-28T06:56:44,149 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family info of region 1588230740 2024-11-28T06:56:44,150 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-28T06:56:44,150 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:56:44,150 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-28T06:56:44,150 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family ns of region 1588230740 2024-11-28T06:56:44,151 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-28T06:56:44,152 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:56:44,152 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-28T06:56:44,152 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family rep_barrier of region 1588230740 2024-11-28T06:56:44,153 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-28T06:56:44,153 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:56:44,153 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-28T06:56:44,153 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family table of region 1588230740 2024-11-28T06:56:44,154 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-28T06:56:44,155 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:56:44,155 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-28T06:56:44,155 DEBUG [RS_OPEN_META-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-28T06:56:44,156 DEBUG [RS_OPEN_META-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7e9a7384-d065-9ff8-196a-59688d3a4920/752619e7-664e-6a55-3bd6-93cba9c9ba92/data/hbase/meta/1588230740 2024-11-28T06:56:44,157 DEBUG [RS_OPEN_META-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7e9a7384-d065-9ff8-196a-59688d3a4920/752619e7-664e-6a55-3bd6-93cba9c9ba92/data/hbase/meta/1588230740 2024-11-28T06:56:44,157 DEBUG [RS_OPEN_META-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-28T06:56:44,157 DEBUG [RS_OPEN_META-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-28T06:56:44,158 DEBUG [RS_OPEN_META-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-28T06:56:44,159 DEBUG [RS_OPEN_META-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-28T06:56:44,160 INFO [RS_OPEN_META-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=74909935, jitterRate=0.11624501645565033}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-28T06:56:44,160 DEBUG [RS_OPEN_META-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-11-28T06:56:44,162 DEBUG [RS_OPEN_META-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1732777004147Writing region info on filesystem at 1732777004148 (+1 ms)Initializing all the Stores at 1732777004148Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732777004148Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732777004149 (+1 ms)Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732777004149Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732777004149Cleaning up temporary data from old regions at 1732777004157 (+8 ms)Running coprocessor post-open hooks at 1732777004160 (+3 ms)Region opened successfully at 1732777004162 (+2 ms) 2024-11-28T06:56:44,169 INFO [RS_OPEN_META-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1732777004074 2024-11-28T06:56:44,180 DEBUG [RS_OPEN_META-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-11-28T06:56:44,181 INFO [RS_OPEN_META-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-11-28T06:56:44,182 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=1cb9cf437a36,35569,1732777001264 2024-11-28T06:56:44,184 INFO [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 1cb9cf437a36,35569,1732777001264, state=OPEN 2024-11-28T06:56:44,211 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37673-0x10181844d810001, quorum=127.0.0.1:58909, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-28T06:56:44,211 DEBUG [pool-21-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35569-0x10181844d810002, quorum=127.0.0.1:58909, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-28T06:56:44,211 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38499-0x10181844d810000, quorum=127.0.0.1:58909, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-28T06:56:44,211 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-28T06:56:44,211 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-28T06:56:44,211 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-28T06:56:44,212 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=1cb9cf437a36,35569,1732777001264 2024-11-28T06:56:44,217 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2 2024-11-28T06:56:44,218 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=1cb9cf437a36,35569,1732777001264 in 304 msec 2024-11-28T06:56:44,224 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1 2024-11-28T06:56:44,224 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 831 msec 2024-11-28T06:56:44,225 DEBUG [PEWorker-2 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-28T06:56:44,226 INFO [PEWorker-2 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces 2024-11-28T06:56:44,246 DEBUG [PEWorker-2 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-28T06:56:44,247 DEBUG [PEWorker-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=1cb9cf437a36,35569,1732777001264, seqNum=-1] 2024-11-28T06:56:44,268 DEBUG [PEWorker-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-28T06:56:44,271 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-4-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40417, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-28T06:56:44,289 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 1.2590 sec 2024-11-28T06:56:44,289 INFO [master/1cb9cf437a36:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1732777004289, completionTime=-1 2024-11-28T06:56:44,292 INFO [master/1cb9cf437a36:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=2; waited=0ms, expected min=2 server(s), max=2 server(s), master is running 2024-11-28T06:56:44,292 DEBUG [master/1cb9cf437a36:0:becomeActiveMaster {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-28T06:56:44,319 INFO [master/1cb9cf437a36:0:becomeActiveMaster {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-11-28T06:56:44,319 INFO [master/1cb9cf437a36:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732777064319 2024-11-28T06:56:44,319 INFO [master/1cb9cf437a36:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732777124319 2024-11-28T06:56:44,319 INFO [master/1cb9cf437a36:0:becomeActiveMaster {}] assignment.AssignmentManager(1783): Joined the cluster in 27 msec 2024-11-28T06:56:44,322 DEBUG [master/1cb9cf437a36:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 1588230740 changed from -1.0 to 0.0, refreshing cache 2024-11-28T06:56:44,329 INFO [master/1cb9cf437a36:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=1cb9cf437a36,38499,1732776999991-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-28T06:56:44,330 INFO [master/1cb9cf437a36:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=1cb9cf437a36,38499,1732776999991-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-28T06:56:44,330 INFO [master/1cb9cf437a36:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=1cb9cf437a36,38499,1732776999991-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-28T06:56:44,332 INFO [master/1cb9cf437a36:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-1cb9cf437a36:38499, period=300000, unit=MILLISECONDS is enabled. 2024-11-28T06:56:44,332 INFO [master/1cb9cf437a36:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2024-11-28T06:56:44,333 INFO [master/1cb9cf437a36:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled. 2024-11-28T06:56:44,340 DEBUG [master/1cb9cf437a36:0.Chore.1 {}] janitor.CatalogJanitor(180): 2024-11-28T06:56:44,363 INFO [master/1cb9cf437a36:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 2.772sec 2024-11-28T06:56:44,365 INFO [master/1cb9cf437a36:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2024-11-28T06:56:44,367 INFO [master/1cb9cf437a36:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2024-11-28T06:56:44,368 INFO [master/1cb9cf437a36:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2024-11-28T06:56:44,368 INFO [master/1cb9cf437a36:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2024-11-28T06:56:44,369 INFO [master/1cb9cf437a36:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2024-11-28T06:56:44,370 INFO [master/1cb9cf437a36:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=1cb9cf437a36,38499,1732776999991-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-28T06:56:44,370 INFO [master/1cb9cf437a36:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=1cb9cf437a36,38499,1732776999991-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2024-11-28T06:56:44,372 DEBUG [master/1cb9cf437a36:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds 2024-11-28T06:56:44,373 INFO [master/1cb9cf437a36:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2024-11-28T06:56:44,373 INFO [master/1cb9cf437a36:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=1cb9cf437a36,38499,1732776999991-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-28T06:56:44,442 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@319823c7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T06:56:44,444 DEBUG [Time-limited test {}] nio.NioEventLoop(110): -Dio.netty.noKeySetOptimization: false 2024-11-28T06:56:44,444 DEBUG [Time-limited test {}] nio.NioEventLoop(111): -Dio.netty.selectorAutoRebuildThreshold: 512 2024-11-28T06:56:44,448 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 1cb9cf437a36,38499,-1 for getting cluster id 2024-11-28T06:56:44,451 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-28T06:56:44,460 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '4cee6445-ce15-4efc-86bd-e2c02b720f01' 2024-11-28T06:56:44,464 DEBUG [RPCClient-NioEventLoopGroup-5-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-28T06:56:44,464 DEBUG [RPCClient-NioEventLoopGroup-5-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "4cee6445-ce15-4efc-86bd-e2c02b720f01" 2024-11-28T06:56:44,466 DEBUG [RPCClient-NioEventLoopGroup-5-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4064e37, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T06:56:44,466 DEBUG [RPCClient-NioEventLoopGroup-5-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [1cb9cf437a36,38499,-1] 2024-11-28T06:56:44,469 DEBUG [RPCClient-NioEventLoopGroup-5-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-28T06:56:44,470 DEBUG [RPCClient-NioEventLoopGroup-5-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T06:56:44,472 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52750, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-28T06:56:44,476 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@19e12cb1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T06:56:44,476 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-28T06:56:44,484 DEBUG [RPCClient-NioEventLoopGroup-5-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=1cb9cf437a36,35569,1732777001264, seqNum=-1] 2024-11-28T06:56:44,485 DEBUG [RPCClient-NioEventLoopGroup-5-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-28T06:56:44,488 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-4-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39476, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-28T06:56:44,511 INFO [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=1cb9cf437a36,38499,1732776999991 2024-11-28T06:56:44,536 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-28T06:56:44,541 DEBUG [RPCClient-NioEventLoopGroup-5-2 {}] client.AsyncConnectionImpl(321): The fetched master address is 1cb9cf437a36,38499,1732776999991 2024-11-28T06:56:44,544 DEBUG [RPCClient-NioEventLoopGroup-5-2 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@73957b38 2024-11-28T06:56:44,545 DEBUG [RPCClient-NioEventLoopGroup-5-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-28T06:56:44,556 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52766, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-28T06:56:44,563 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38499 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'testBlockEvictionOnRegionMove1732777004511', {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-28T06:56:44,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38499 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=testBlockEvictionOnRegionMove1732777004511 2024-11-28T06:56:44,574 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=testBlockEvictionOnRegionMove1732777004511 execute state=CREATE_TABLE_PRE_OPERATION 2024-11-28T06:56:44,577 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38499 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "testBlockEvictionOnRegionMove1732777004511" procId is: 4 2024-11-28T06:56:44,577 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:56:44,580 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=testBlockEvictionOnRegionMove1732777004511 execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-28T06:56:44,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38499 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-28T06:56:44,609 INFO [RegionOpenAndInit-testBlockEvictionOnRegionMove1732777004511-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 9dd73ea1b712ae668b4c7cba5ad52740, NAME => 'testBlockEvictionOnRegionMove1732777004511,,1732777004558.9dd73ea1b712ae668b4c7cba5ad52740.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testBlockEvictionOnRegionMove1732777004511', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7e9a7384-d065-9ff8-196a-59688d3a4920/752619e7-664e-6a55-3bd6-93cba9c9ba92 2024-11-28T06:56:44,616 DEBUG [RegionOpenAndInit-testBlockEvictionOnRegionMove1732777004511-pool-0 {}] regionserver.HRegion(898): Instantiated testBlockEvictionOnRegionMove1732777004511,,1732777004558.9dd73ea1b712ae668b4c7cba5ad52740.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-28T06:56:44,616 DEBUG [RegionOpenAndInit-testBlockEvictionOnRegionMove1732777004511-pool-0 {}] regionserver.HRegion(1722): Closing 9dd73ea1b712ae668b4c7cba5ad52740, disabling compactions & flushes 2024-11-28T06:56:44,617 INFO [RegionOpenAndInit-testBlockEvictionOnRegionMove1732777004511-pool-0 {}] regionserver.HRegion(1755): Closing region testBlockEvictionOnRegionMove1732777004511,,1732777004558.9dd73ea1b712ae668b4c7cba5ad52740. 2024-11-28T06:56:44,617 DEBUG [RegionOpenAndInit-testBlockEvictionOnRegionMove1732777004511-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on testBlockEvictionOnRegionMove1732777004511,,1732777004558.9dd73ea1b712ae668b4c7cba5ad52740. 2024-11-28T06:56:44,617 DEBUG [RegionOpenAndInit-testBlockEvictionOnRegionMove1732777004511-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on testBlockEvictionOnRegionMove1732777004511,,1732777004558.9dd73ea1b712ae668b4c7cba5ad52740. after waiting 0 ms 2024-11-28T06:56:44,617 DEBUG [RegionOpenAndInit-testBlockEvictionOnRegionMove1732777004511-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region testBlockEvictionOnRegionMove1732777004511,,1732777004558.9dd73ea1b712ae668b4c7cba5ad52740. 2024-11-28T06:56:44,617 INFO [RegionOpenAndInit-testBlockEvictionOnRegionMove1732777004511-pool-0 {}] regionserver.HRegion(1973): Closed testBlockEvictionOnRegionMove1732777004511,,1732777004558.9dd73ea1b712ae668b4c7cba5ad52740. 2024-11-28T06:56:44,617 DEBUG [RegionOpenAndInit-testBlockEvictionOnRegionMove1732777004511-pool-0 {}] regionserver.HRegion(1676): Region close journal for 9dd73ea1b712ae668b4c7cba5ad52740: Waiting for close lock at 1732777004616Disabling compacts and flushes for region at 1732777004616Disabling writes for close at 1732777004617 (+1 ms)Writing region close event to WAL at 1732777004617Closed at 1732777004617 2024-11-28T06:56:44,619 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=testBlockEvictionOnRegionMove1732777004511 execute state=CREATE_TABLE_ADD_TO_META 2024-11-28T06:56:44,624 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testBlockEvictionOnRegionMove1732777004511,,1732777004558.9dd73ea1b712ae668b4c7cba5ad52740.","families":{"info":[{"qualifier":"regioninfo","vlen":76,"tag":[],"timestamp":"1732777004619"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732777004619"}]},"ts":"1732777004619"} 2024-11-28T06:56:44,629 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-28T06:56:44,630 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=testBlockEvictionOnRegionMove1732777004511 execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-28T06:56:44,633 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testBlockEvictionOnRegionMove1732777004511","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732777004630"}]},"ts":"1732777004630"} 2024-11-28T06:56:44,637 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=testBlockEvictionOnRegionMove1732777004511, state=ENABLING in hbase:meta 2024-11-28T06:56:44,638 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(204): Hosts are {1cb9cf437a36=0} racks are {/default-rack=0} 2024-11-28T06:56:44,639 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-11-28T06:56:44,639 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-11-28T06:56:44,639 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-11-28T06:56:44,639 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-11-28T06:56:44,639 INFO [PEWorker-3 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-11-28T06:56:44,639 INFO [PEWorker-3 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-11-28T06:56:44,639 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-11-28T06:56:44,640 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1732777004511, region=9dd73ea1b712ae668b4c7cba5ad52740, ASSIGN}] 2024-11-28T06:56:44,643 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1732777004511, region=9dd73ea1b712ae668b4c7cba5ad52740, ASSIGN 2024-11-28T06:56:44,647 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1732777004511, region=9dd73ea1b712ae668b4c7cba5ad52740, ASSIGN; state=OFFLINE, location=1cb9cf437a36,37673,1732777001059; forceNewPlan=false, retain=false 2024-11-28T06:56:44,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38499 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-28T06:56:44,800 INFO [1cb9cf437a36:38499 {}] balancer.BaseLoadBalancer(388): Reassigned 1 regions. 1 retained the pre-restart assignment. 2024-11-28T06:56:44,800 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=9dd73ea1b712ae668b4c7cba5ad52740, regionState=OPENING, regionLocation=1cb9cf437a36,37673,1732777001059 2024-11-28T06:56:44,804 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-4-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1732777004511, region=9dd73ea1b712ae668b4c7cba5ad52740, ASSIGN because future has completed 2024-11-28T06:56:44,805 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 9dd73ea1b712ae668b4c7cba5ad52740, server=1cb9cf437a36,37673,1732777001059}] 2024-11-28T06:56:44,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38499 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-28T06:56:44,959 DEBUG [RSProcedureDispatcher-pool-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-28T06:56:44,962 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43513, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-28T06:56:44,965 INFO [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(132): Open testBlockEvictionOnRegionMove1732777004511,,1732777004558.9dd73ea1b712ae668b4c7cba5ad52740. 2024-11-28T06:56:44,966 DEBUG [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7752): Opening region: {ENCODED => 9dd73ea1b712ae668b4c7cba5ad52740, NAME => 'testBlockEvictionOnRegionMove1732777004511,,1732777004558.9dd73ea1b712ae668b4c7cba5ad52740.', STARTKEY => '', ENDKEY => ''} 2024-11-28T06:56:44,967 DEBUG [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testBlockEvictionOnRegionMove1732777004511 9dd73ea1b712ae668b4c7cba5ad52740 2024-11-28T06:56:44,967 DEBUG [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(898): Instantiated testBlockEvictionOnRegionMove1732777004511,,1732777004558.9dd73ea1b712ae668b4c7cba5ad52740.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-28T06:56:44,967 DEBUG [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7794): checking encryption for 9dd73ea1b712ae668b4c7cba5ad52740 2024-11-28T06:56:44,967 DEBUG [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7797): checking classloading for 9dd73ea1b712ae668b4c7cba5ad52740 2024-11-28T06:56:44,969 INFO [StoreOpener-9dd73ea1b712ae668b4c7cba5ad52740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family family of region 9dd73ea1b712ae668b4c7cba5ad52740 2024-11-28T06:56:44,980 INFO [StoreOpener-9dd73ea1b712ae668b4c7cba5ad52740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 9dd73ea1b712ae668b4c7cba5ad52740 columnFamilyName family 2024-11-28T06:56:44,980 DEBUG [StoreOpener-9dd73ea1b712ae668b4c7cba5ad52740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:56:44,981 INFO [StoreOpener-9dd73ea1b712ae668b4c7cba5ad52740-1 {}] regionserver.HStore(327): Store=9dd73ea1b712ae668b4c7cba5ad52740/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-28T06:56:44,981 DEBUG [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1038): replaying wal for 9dd73ea1b712ae668b4c7cba5ad52740 2024-11-28T06:56:44,982 DEBUG [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7e9a7384-d065-9ff8-196a-59688d3a4920/752619e7-664e-6a55-3bd6-93cba9c9ba92/data/default/testBlockEvictionOnRegionMove1732777004511/9dd73ea1b712ae668b4c7cba5ad52740 2024-11-28T06:56:44,982 DEBUG [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7e9a7384-d065-9ff8-196a-59688d3a4920/752619e7-664e-6a55-3bd6-93cba9c9ba92/data/default/testBlockEvictionOnRegionMove1732777004511/9dd73ea1b712ae668b4c7cba5ad52740 2024-11-28T06:56:44,983 DEBUG [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1048): stopping wal replay for 9dd73ea1b712ae668b4c7cba5ad52740 2024-11-28T06:56:44,983 DEBUG [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1060): Cleaning up temporary data for 9dd73ea1b712ae668b4c7cba5ad52740 2024-11-28T06:56:44,984 DEBUG [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1093): writing seq id for 9dd73ea1b712ae668b4c7cba5ad52740 2024-11-28T06:56:44,991 DEBUG [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7e9a7384-d065-9ff8-196a-59688d3a4920/752619e7-664e-6a55-3bd6-93cba9c9ba92/data/default/testBlockEvictionOnRegionMove1732777004511/9dd73ea1b712ae668b4c7cba5ad52740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-28T06:56:44,992 INFO [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1114): Opened 9dd73ea1b712ae668b4c7cba5ad52740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=62218346, jitterRate=-0.07287439703941345}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-28T06:56:44,992 DEBUG [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 9dd73ea1b712ae668b4c7cba5ad52740 2024-11-28T06:56:44,993 DEBUG [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1006): Region open journal for 9dd73ea1b712ae668b4c7cba5ad52740: Running coprocessor pre-open hook at 1732777004968Writing region info on filesystem at 1732777004968Initializing all the Stores at 1732777004968Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732777004968Cleaning up temporary data from old regions at 1732777004983 (+15 ms)Running coprocessor post-open hooks at 1732777004992 (+9 ms)Region opened successfully at 1732777004993 (+1 ms) 2024-11-28T06:56:44,994 INFO [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2236): Post open deploy tasks for testBlockEvictionOnRegionMove1732777004511,,1732777004558.9dd73ea1b712ae668b4c7cba5ad52740., pid=6, masterSystemTime=1732777004959 2024-11-28T06:56:44,998 DEBUG [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2266): Finished post open deploy task for testBlockEvictionOnRegionMove1732777004511,,1732777004558.9dd73ea1b712ae668b4c7cba5ad52740. 2024-11-28T06:56:44,998 INFO [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(153): Opened testBlockEvictionOnRegionMove1732777004511,,1732777004558.9dd73ea1b712ae668b4c7cba5ad52740. 2024-11-28T06:56:44,998 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=9dd73ea1b712ae668b4c7cba5ad52740, regionState=OPEN, openSeqNum=2, regionLocation=1cb9cf437a36,37673,1732777001059 2024-11-28T06:56:45,003 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-4-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 9dd73ea1b712ae668b4c7cba5ad52740, server=1cb9cf437a36,37673,1732777001059 because future has completed 2024-11-28T06:56:45,007 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=6, resume processing ppid=5 2024-11-28T06:56:45,007 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure 9dd73ea1b712ae668b4c7cba5ad52740, server=1cb9cf437a36,37673,1732777001059 in 199 msec 2024-11-28T06:56:45,010 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=5, resume processing ppid=4 2024-11-28T06:56:45,010 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1732777004511, region=9dd73ea1b712ae668b4c7cba5ad52740, ASSIGN in 367 msec 2024-11-28T06:56:45,010 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=testBlockEvictionOnRegionMove1732777004511 execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-28T06:56:45,011 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testBlockEvictionOnRegionMove1732777004511","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732777005011"}]},"ts":"1732777005011"} 2024-11-28T06:56:45,014 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=testBlockEvictionOnRegionMove1732777004511, state=ENABLED in hbase:meta 2024-11-28T06:56:45,016 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=testBlockEvictionOnRegionMove1732777004511 execute state=CREATE_TABLE_POST_OPERATION 2024-11-28T06:56:45,018 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=testBlockEvictionOnRegionMove1732777004511 in 450 msec 2024-11-28T06:56:45,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38499 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-28T06:56:45,212 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table testBlockEvictionOnRegionMove1732777004511 get assigned. Timeout = 60000ms 2024-11-28T06:56:45,212 INFO [RPCClient-NioEventLoopGroup-5-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:testBlockEvictionOnRegionMove1732777004511 completed 2024-11-28T06:56:45,215 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-28T06:56:45,222 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table testBlockEvictionOnRegionMove1732777004511 assigned to meta. Checking AM states. 2024-11-28T06:56:45,223 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-28T06:56:45,223 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table testBlockEvictionOnRegionMove1732777004511 assigned. 2024-11-28T06:56:45,233 DEBUG [RPCClient-NioEventLoopGroup-5-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testBlockEvictionOnRegionMove1732777004511', row='row1', locateType=CURRENT is [region=testBlockEvictionOnRegionMove1732777004511,,1732777004558.9dd73ea1b712ae668b4c7cba5ad52740., hostname=1cb9cf437a36,37673,1732777001059, seqNum=2] 2024-11-28T06:56:45,235 DEBUG [RPCClient-NioEventLoopGroup-5-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-28T06:56:45,239 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57632, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-28T06:56:45,250 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 9dd73ea1b712ae668b4c7cba5ad52740 1/1 column families, dataSize=78 B heapSize=480 B 2024-11-28T06:56:45,280 DEBUG [Time-limited test {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 2a9a061c857d42db9c7923afadf27520 2024-11-28T06:56:45,322 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7e9a7384-d065-9ff8-196a-59688d3a4920/752619e7-664e-6a55-3bd6-93cba9c9ba92/data/default/testBlockEvictionOnRegionMove1732777004511/9dd73ea1b712ae668b4c7cba5ad52740/.tmp/family/2a9a061c857d42db9c7923afadf27520 is 43, key is row1/family:qf1/1/Put/seqid=0 2024-11-28T06:56:45,327 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=78 B at sequenceid=6 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7e9a7384-d065-9ff8-196a-59688d3a4920/752619e7-664e-6a55-3bd6-93cba9c9ba92/data/default/testBlockEvictionOnRegionMove1732777004511/9dd73ea1b712ae668b4c7cba5ad52740/.tmp/family/2a9a061c857d42db9c7923afadf27520 2024-11-28T06:56:45,364 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7e9a7384-d065-9ff8-196a-59688d3a4920/752619e7-664e-6a55-3bd6-93cba9c9ba92/data/default/testBlockEvictionOnRegionMove1732777004511/9dd73ea1b712ae668b4c7cba5ad52740/.tmp/family/2a9a061c857d42db9c7923afadf27520 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7e9a7384-d065-9ff8-196a-59688d3a4920/752619e7-664e-6a55-3bd6-93cba9c9ba92/data/default/testBlockEvictionOnRegionMove1732777004511/9dd73ea1b712ae668b4c7cba5ad52740/family/2a9a061c857d42db9c7923afadf27520 2024-11-28T06:56:45,366 DEBUG [Time-limited test {}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7e9a7384-d065-9ff8-196a-59688d3a4920/752619e7-664e-6a55-3bd6-93cba9c9ba92/data/default/testBlockEvictionOnRegionMove1732777004511/9dd73ea1b712ae668b4c7cba5ad52740/family/2a9a061c857d42db9c7923afadf27520, delay=968 ms 2024-11-28T06:56:45,368 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7e9a7384-d065-9ff8-196a-59688d3a4920/752619e7-664e-6a55-3bd6-93cba9c9ba92/data/default/testBlockEvictionOnRegionMove1732777004511/9dd73ea1b712ae668b4c7cba5ad52740/family/2a9a061c857d42db9c7923afadf27520, entries=2, sequenceid=6, filesize=4.9 K 2024-11-28T06:56:45,375 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~78 B/78, heapSize ~464 B/464, currentSize=0 B/0 for 9dd73ea1b712ae668b4c7cba5ad52740 in 124ms, sequenceid=6, compaction requested=false 2024-11-28T06:56:45,376 DEBUG [Time-limited test {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'testBlockEvictionOnRegionMove1732777004511' 2024-11-28T06:56:45,378 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for 9dd73ea1b712ae668b4c7cba5ad52740: 2024-11-28T06:56:46,336 DEBUG [hfile-prefetch-1732777005366 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 2a9a061c857d42db9c7923afadf27520_0. 2024-11-28T06:56:46,340 DEBUG [hfile-prefetch-1732777005366 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 2a9a061c857d42db9c7923afadf27520_117. 2024-11-28T06:56:46,342 DEBUG [hfile-prefetch-1732777005366 {}] bucket.BucketCache(2281): Notifying caching completed for file file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7e9a7384-d065-9ff8-196a-59688d3a4920/752619e7-664e-6a55-3bd6-93cba9c9ba92/data/default/testBlockEvictionOnRegionMove1732777004511/9dd73ea1b712ae668b4c7cba5ad52740/family/2a9a061c857d42db9c7923afadf27520, with total blocks 2, and data blocks 1 2024-11-28T06:56:46,342 DEBUG [hfile-prefetch-1732777005366 {}] bucket.BucketCache(2285): iterating over 0 entries in the backing map 2024-11-28T06:56:46,342 DEBUG [hfile-prefetch-1732777005366 {}] bucket.BucketCache(2308): Prefetch executor completed for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7e9a7384-d065-9ff8-196a-59688d3a4920/752619e7-664e-6a55-3bd6-93cba9c9ba92/data/default/testBlockEvictionOnRegionMove1732777004511/9dd73ea1b712ae668b4c7cba5ad52740/family/2a9a061c857d42db9c7923afadf27520, but only 0 data blocks were cached. Total data blocks for file: 1. Checking for blocks pending cache in cache writer queue. 2024-11-28T06:56:46,343 DEBUG [hfile-prefetch-1732777005366 {}] bucket.BucketCache(2318): There are still blocks pending caching for file file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7e9a7384-d065-9ff8-196a-59688d3a4920/752619e7-664e-6a55-3bd6-93cba9c9ba92/data/default/testBlockEvictionOnRegionMove1732777004511/9dd73ea1b712ae668b4c7cba5ad52740/family/2a9a061c857d42db9c7923afadf27520. Will sleep 100ms and try the verification again. 2024-11-28T06:56:46,390 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38499 {}] master.HMaster(2410): Client=jenkins//172.17.0.2 move hri=9dd73ea1b712ae668b4c7cba5ad52740, source=1cb9cf437a36,37673,1732777001059, destination=1cb9cf437a36,35569,1732777001264, warming up region on 1cb9cf437a36,35569,1732777001264 2024-11-28T06:56:46,392 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38499 {}] master.HMaster(2414): Client=jenkins//172.17.0.2 move hri=9dd73ea1b712ae668b4c7cba5ad52740, source=1cb9cf437a36,37673,1732777001059, destination=1cb9cf437a36,35569,1732777001264, running balancer 2024-11-28T06:56:46,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38499 {}] procedure2.ProcedureExecutor(1139): Stored pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1732777004511, region=9dd73ea1b712ae668b4c7cba5ad52740, REOPEN/MOVE 2024-11-28T06:56:46,394 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35569 {}] regionserver.RSRpcServices(2066): Warmup testBlockEvictionOnRegionMove1732777004511,,1732777004558.9dd73ea1b712ae668b4c7cba5ad52740. 2024-11-28T06:56:46,394 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1732777004511, region=9dd73ea1b712ae668b4c7cba5ad52740, REOPEN/MOVE 2024-11-28T06:56:46,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35569 {}] regionserver.HRegion(7855): Warmup {ENCODED => 9dd73ea1b712ae668b4c7cba5ad52740, NAME => 'testBlockEvictionOnRegionMove1732777004511,,1732777004558.9dd73ea1b712ae668b4c7cba5ad52740.', STARTKEY => '', ENDKEY => ''} 2024-11-28T06:56:46,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35569 {}] regionserver.HRegion(898): Instantiated testBlockEvictionOnRegionMove1732777004511,,1732777004558.9dd73ea1b712ae668b4c7cba5ad52740.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-28T06:56:46,396 INFO [StoreOpener-9dd73ea1b712ae668b4c7cba5ad52740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family family of region 9dd73ea1b712ae668b4c7cba5ad52740 2024-11-28T06:56:46,396 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=9dd73ea1b712ae668b4c7cba5ad52740, regionState=CLOSING, regionLocation=1cb9cf437a36,37673,1732777001059 2024-11-28T06:56:46,397 INFO [StoreOpener-9dd73ea1b712ae668b4c7cba5ad52740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 9dd73ea1b712ae668b4c7cba5ad52740 columnFamilyName family 2024-11-28T06:56:46,397 DEBUG [StoreOpener-9dd73ea1b712ae668b4c7cba5ad52740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:56:46,401 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-4-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1732777004511, region=9dd73ea1b712ae668b4c7cba5ad52740, REOPEN/MOVE because future has completed 2024-11-28T06:56:46,402 DEBUG [StoreOpener-9dd73ea1b712ae668b4c7cba5ad52740-1 {}] regionserver.StoreEngine(278): loaded file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7e9a7384-d065-9ff8-196a-59688d3a4920/752619e7-664e-6a55-3bd6-93cba9c9ba92/data/default/testBlockEvictionOnRegionMove1732777004511/9dd73ea1b712ae668b4c7cba5ad52740/family/2a9a061c857d42db9c7923afadf27520 2024-11-28T06:56:46,402 INFO [StoreOpener-9dd73ea1b712ae668b4c7cba5ad52740-1 {}] regionserver.HStore(327): Store=9dd73ea1b712ae668b4c7cba5ad52740/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-28T06:56:46,402 DEBUG [PEWorker-4 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: true 2024-11-28T06:56:46,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35569 {}] regionserver.HRegion(1722): Closing 9dd73ea1b712ae668b4c7cba5ad52740, disabling compactions & flushes 2024-11-28T06:56:46,402 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35569 {}] regionserver.HRegion(1755): Closing region testBlockEvictionOnRegionMove1732777004511,,1732777004558.9dd73ea1b712ae668b4c7cba5ad52740. 2024-11-28T06:56:46,402 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=8, ppid=7, state=RUNNABLE, hasLock=false; CloseRegionProcedure 9dd73ea1b712ae668b4c7cba5ad52740, server=1cb9cf437a36,37673,1732777001059}] 2024-11-28T06:56:46,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35569 {}] regionserver.HRegion(1776): Time limited wait for close lock on testBlockEvictionOnRegionMove1732777004511,,1732777004558.9dd73ea1b712ae668b4c7cba5ad52740. 2024-11-28T06:56:46,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35569 {}] regionserver.HRegion(1843): Acquired close lock on testBlockEvictionOnRegionMove1732777004511,,1732777004558.9dd73ea1b712ae668b4c7cba5ad52740. after waiting 0 ms 2024-11-28T06:56:46,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35569 {}] regionserver.HRegion(1853): Updates disabled for region testBlockEvictionOnRegionMove1732777004511,,1732777004558.9dd73ea1b712ae668b4c7cba5ad52740. 2024-11-28T06:56:46,408 DEBUG [StoreFileCloser-9dd73ea1b712ae668b4c7cba5ad52740-family-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7e9a7384-d065-9ff8-196a-59688d3a4920/752619e7-664e-6a55-3bd6-93cba9c9ba92/data/default/testBlockEvictionOnRegionMove1732777004511/9dd73ea1b712ae668b4c7cba5ad52740/family/2a9a061c857d42db9c7923afadf27520 2024-11-28T06:56:46,410 WARN [hfile-prefetch-1732777005366 {}] hfile.HFilePreadReader$1(158): Prefetch path=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7e9a7384-d065-9ff8-196a-59688d3a4920/752619e7-664e-6a55-3bd6-93cba9c9ba92/data/default/testBlockEvictionOnRegionMove1732777004511/9dd73ea1b712ae668b4c7cba5ad52740/family/2a9a061c857d42db9c7923afadf27520, offset=158, end=158 java.lang.RuntimeException: java.lang.InterruptedException: sleep interrupted at org.apache.hadoop.hbase.io.hfile.bucket.BucketCache.notifyFileCachingCompleted(BucketCache.java:2337) ~[classes/:?] at org.apache.hadoop.hbase.io.hfile.CombinedBlockCache.notifyFileCachingCompleted(CombinedBlockCache.java:469) ~[classes/:?] at org.apache.hadoop.hbase.io.hfile.HFilePreadReader$1.run(HFilePreadReader.java:148) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.lambda$tracedRunnable$2(TraceUtil.java:155) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:539) ~[?:?] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:304) ~[?:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCache.notifyFileCachingCompleted(BucketCache.java:2320) ~[classes/:?] ... 9 more 2024-11-28T06:56:46,411 DEBUG [hfile-prefetch-1732777005366 {}] hfile.PrefetchExecutor(121): Prefetch completed for 2a9a061c857d42db9c7923afadf27520 2024-11-28T06:56:46,412 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35569 {}] regionserver.HRegion(1973): Closed testBlockEvictionOnRegionMove1732777004511,,1732777004558.9dd73ea1b712ae668b4c7cba5ad52740. 2024-11-28T06:56:46,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35569 {}] regionserver.HRegion(1676): Region close journal for 9dd73ea1b712ae668b4c7cba5ad52740: Waiting for close lock at 1732777006402Disabling compacts and flushes for region at 1732777006402Disabling writes for close at 1732777006403 (+1 ms)Writing region close event to WAL at 1732777006412 (+9 ms)Closed at 1732777006412 2024-11-28T06:56:46,436 DEBUG [bucket-cache-persister {}] bucket.BucketCachePersister(47): Cache is inconsistent, persisting to disk 2024-11-28T06:56:46,436 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1487): Thread bucket-cache-persister started persisting bucket cache to file 2024-11-28T06:56:46,437 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1494): Persist in new chunked persistence format. 2024-11-28T06:56:46,437 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1673): persistToFile: before persisting backing map size: 0, fullycachedFiles size: 0, chunkSize: 10000 2024-11-28T06:56:46,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38499 {}] procedure.ProcedureSyncWait(219): waitFor pid=7 2024-11-28T06:56:46,456 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1680): persistToFile: after persisting backing map size: 0, fullycachedFiles size: 0 2024-11-28T06:56:46,456 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1498): PersistToFile: after persisting backing map size: 0, fullycachedFiles size: 0, file name: bucket.persistence1732777006436 2024-11-28T06:56:46,456 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1509): Thread bucket-cache-persister finished persisting bucket cache to file, renaming 2024-11-28T06:56:46,563 INFO [RS_CLOSE_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] handler.UnassignRegionHandler(122): Close 9dd73ea1b712ae668b4c7cba5ad52740 2024-11-28T06:56:46,563 DEBUG [RS_CLOSE_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: true 2024-11-28T06:56:46,564 DEBUG [RS_CLOSE_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1722): Closing 9dd73ea1b712ae668b4c7cba5ad52740, disabling compactions & flushes 2024-11-28T06:56:46,564 INFO [RS_CLOSE_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1755): Closing region testBlockEvictionOnRegionMove1732777004511,,1732777004558.9dd73ea1b712ae668b4c7cba5ad52740. 2024-11-28T06:56:46,564 DEBUG [RS_CLOSE_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1776): Time limited wait for close lock on testBlockEvictionOnRegionMove1732777004511,,1732777004558.9dd73ea1b712ae668b4c7cba5ad52740. 2024-11-28T06:56:46,564 DEBUG [RS_CLOSE_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1843): Acquired close lock on testBlockEvictionOnRegionMove1732777004511,,1732777004558.9dd73ea1b712ae668b4c7cba5ad52740. after waiting 0 ms 2024-11-28T06:56:46,564 DEBUG [RS_CLOSE_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1853): Updates disabled for region testBlockEvictionOnRegionMove1732777004511,,1732777004558.9dd73ea1b712ae668b4c7cba5ad52740. 2024-11-28T06:56:46,566 DEBUG [StoreFileCloser-9dd73ea1b712ae668b4c7cba5ad52740-family-1 {}] bucket.BucketCache(1863): found 1 blocks for file 2a9a061c857d42db9c7923afadf27520, starting offset: 0, end offset: 9223372036854775807 2024-11-28T06:56:46,568 DEBUG [StoreFileCloser-9dd73ea1b712ae668b4c7cba5ad52740-family-1 {}] bucket.BucketCache(856): removed key 2a9a061c857d42db9c7923afadf27520_0 from back map with offset lock 2094080 in the evict process 2024-11-28T06:56:46,573 DEBUG [RS_CLOSE_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7e9a7384-d065-9ff8-196a-59688d3a4920/752619e7-664e-6a55-3bd6-93cba9c9ba92/data/default/testBlockEvictionOnRegionMove1732777004511/9dd73ea1b712ae668b4c7cba5ad52740/recovered.edits/9.seqid, newMaxSeqId=9, maxSeqId=1 2024-11-28T06:56:46,575 INFO [RS_CLOSE_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1973): Closed testBlockEvictionOnRegionMove1732777004511,,1732777004558.9dd73ea1b712ae668b4c7cba5ad52740. 2024-11-28T06:56:46,575 DEBUG [RS_CLOSE_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1676): Region close journal for 9dd73ea1b712ae668b4c7cba5ad52740: Waiting for close lock at 1732777006564Running coprocessor pre-close hooks at 1732777006564Disabling compacts and flushes for region at 1732777006564Disabling writes for close at 1732777006564Writing region close event to WAL at 1732777006568 (+4 ms)Running coprocessor post-close hooks at 1732777006574 (+6 ms)Closed at 1732777006575 (+1 ms) 2024-11-28T06:56:46,576 INFO [RS_CLOSE_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegionServer(3302): Adding 9dd73ea1b712ae668b4c7cba5ad52740 move to 1cb9cf437a36,35569,1732777001264 record at close sequenceid=6 2024-11-28T06:56:46,581 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=9dd73ea1b712ae668b4c7cba5ad52740, regionState=CLOSED 2024-11-28T06:56:46,581 INFO [RS_CLOSE_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] handler.UnassignRegionHandler(157): Closed 9dd73ea1b712ae668b4c7cba5ad52740 2024-11-28T06:56:46,585 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-4-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=8, ppid=7, state=RUNNABLE, hasLock=false; CloseRegionProcedure 9dd73ea1b712ae668b4c7cba5ad52740, server=1cb9cf437a36,37673,1732777001059 because future has completed 2024-11-28T06:56:46,590 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=8, resume processing ppid=7 2024-11-28T06:56:46,591 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=8, ppid=7, state=SUCCESS, hasLock=false; CloseRegionProcedure 9dd73ea1b712ae668b4c7cba5ad52740, server=1cb9cf437a36,37673,1732777001059 in 186 msec 2024-11-28T06:56:46,591 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1732777004511, region=9dd73ea1b712ae668b4c7cba5ad52740, REOPEN/MOVE; state=CLOSED, location=1cb9cf437a36,35569,1732777001264; forceNewPlan=false, retain=false 2024-11-28T06:56:46,599 DEBUG [bucket-cache-persister {}] bucket.BucketCachePersister(47): Cache is inconsistent, persisting to disk 2024-11-28T06:56:46,599 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1487): Thread bucket-cache-persister started persisting bucket cache to file 2024-11-28T06:56:46,599 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1494): Persist in new chunked persistence format. 2024-11-28T06:56:46,600 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1673): persistToFile: before persisting backing map size: 0, fullycachedFiles size: 0, chunkSize: 10000 2024-11-28T06:56:46,603 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1680): persistToFile: after persisting backing map size: 0, fullycachedFiles size: 0 2024-11-28T06:56:46,603 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1498): PersistToFile: after persisting backing map size: 0, fullycachedFiles size: 0, file name: bucket.persistence1732777006599 2024-11-28T06:56:46,603 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1509): Thread bucket-cache-persister finished persisting bucket cache to file, renaming 2024-11-28T06:56:46,742 INFO [1cb9cf437a36:38499 {}] balancer.BaseLoadBalancer(388): Reassigned 1 regions. 1 retained the pre-restart assignment. 2024-11-28T06:56:46,743 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=9dd73ea1b712ae668b4c7cba5ad52740, regionState=OPENING, regionLocation=1cb9cf437a36,35569,1732777001264 2024-11-28T06:56:46,746 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-4-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1732777004511, region=9dd73ea1b712ae668b4c7cba5ad52740, REOPEN/MOVE because future has completed 2024-11-28T06:56:46,747 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=9, ppid=7, state=RUNNABLE, hasLock=false; OpenRegionProcedure 9dd73ea1b712ae668b4c7cba5ad52740, server=1cb9cf437a36,35569,1732777001264}] 2024-11-28T06:56:46,902 INFO [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] handler.AssignRegionHandler(132): Open testBlockEvictionOnRegionMove1732777004511,,1732777004558.9dd73ea1b712ae668b4c7cba5ad52740. 2024-11-28T06:56:46,903 DEBUG [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(7752): Opening region: {ENCODED => 9dd73ea1b712ae668b4c7cba5ad52740, NAME => 'testBlockEvictionOnRegionMove1732777004511,,1732777004558.9dd73ea1b712ae668b4c7cba5ad52740.', STARTKEY => '', ENDKEY => ''} 2024-11-28T06:56:46,903 DEBUG [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testBlockEvictionOnRegionMove1732777004511 9dd73ea1b712ae668b4c7cba5ad52740 2024-11-28T06:56:46,903 DEBUG [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(898): Instantiated testBlockEvictionOnRegionMove1732777004511,,1732777004558.9dd73ea1b712ae668b4c7cba5ad52740.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-28T06:56:46,903 DEBUG [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(7794): checking encryption for 9dd73ea1b712ae668b4c7cba5ad52740 2024-11-28T06:56:46,903 DEBUG [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(7797): checking classloading for 9dd73ea1b712ae668b4c7cba5ad52740 2024-11-28T06:56:46,904 INFO [StoreOpener-9dd73ea1b712ae668b4c7cba5ad52740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family family of region 9dd73ea1b712ae668b4c7cba5ad52740 2024-11-28T06:56:46,905 INFO [StoreOpener-9dd73ea1b712ae668b4c7cba5ad52740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 9dd73ea1b712ae668b4c7cba5ad52740 columnFamilyName family 2024-11-28T06:56:46,906 DEBUG [StoreOpener-9dd73ea1b712ae668b4c7cba5ad52740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:56:46,907 DEBUG [StoreFileOpener-9dd73ea1b712ae668b4c7cba5ad52740-family-1 {}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7e9a7384-d065-9ff8-196a-59688d3a4920/752619e7-664e-6a55-3bd6-93cba9c9ba92/data/default/testBlockEvictionOnRegionMove1732777004511/9dd73ea1b712ae668b4c7cba5ad52740/family/2a9a061c857d42db9c7923afadf27520, delay=992 ms 2024-11-28T06:56:46,909 DEBUG [StoreOpener-9dd73ea1b712ae668b4c7cba5ad52740-1 {}] regionserver.StoreEngine(278): loaded file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7e9a7384-d065-9ff8-196a-59688d3a4920/752619e7-664e-6a55-3bd6-93cba9c9ba92/data/default/testBlockEvictionOnRegionMove1732777004511/9dd73ea1b712ae668b4c7cba5ad52740/family/2a9a061c857d42db9c7923afadf27520 2024-11-28T06:56:46,910 INFO [StoreOpener-9dd73ea1b712ae668b4c7cba5ad52740-1 {}] regionserver.HStore(327): Store=9dd73ea1b712ae668b4c7cba5ad52740/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-28T06:56:46,910 DEBUG [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1038): replaying wal for 9dd73ea1b712ae668b4c7cba5ad52740 2024-11-28T06:56:46,911 DEBUG [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7e9a7384-d065-9ff8-196a-59688d3a4920/752619e7-664e-6a55-3bd6-93cba9c9ba92/data/default/testBlockEvictionOnRegionMove1732777004511/9dd73ea1b712ae668b4c7cba5ad52740 2024-11-28T06:56:46,911 DEBUG [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7e9a7384-d065-9ff8-196a-59688d3a4920/752619e7-664e-6a55-3bd6-93cba9c9ba92/data/default/testBlockEvictionOnRegionMove1732777004511/9dd73ea1b712ae668b4c7cba5ad52740 2024-11-28T06:56:46,911 DEBUG [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1048): stopping wal replay for 9dd73ea1b712ae668b4c7cba5ad52740 2024-11-28T06:56:46,911 DEBUG [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1060): Cleaning up temporary data for 9dd73ea1b712ae668b4c7cba5ad52740 2024-11-28T06:56:46,917 DEBUG [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1093): writing seq id for 9dd73ea1b712ae668b4c7cba5ad52740 2024-11-28T06:56:46,918 INFO [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1114): Opened 9dd73ea1b712ae668b4c7cba5ad52740; next sequenceid=10; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=58984083, jitterRate=-0.12106867134571075}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-28T06:56:46,918 DEBUG [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 9dd73ea1b712ae668b4c7cba5ad52740 2024-11-28T06:56:46,920 DEBUG [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1006): Region open journal for 9dd73ea1b712ae668b4c7cba5ad52740: Running coprocessor pre-open hook at 1732777006904Writing region info on filesystem at 1732777006904Initializing all the Stores at 1732777006904Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732777006904Cleaning up temporary data from old regions at 1732777006911 (+7 ms)Running coprocessor post-open hooks at 1732777006918 (+7 ms)Region opened successfully at 1732777006919 (+1 ms) 2024-11-28T06:56:46,921 INFO [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegionServer(2236): Post open deploy tasks for testBlockEvictionOnRegionMove1732777004511,,1732777004558.9dd73ea1b712ae668b4c7cba5ad52740., pid=9, masterSystemTime=1732777006900 2024-11-28T06:56:46,924 DEBUG [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegionServer(2266): Finished post open deploy task for testBlockEvictionOnRegionMove1732777004511,,1732777004558.9dd73ea1b712ae668b4c7cba5ad52740. 2024-11-28T06:56:46,924 INFO [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] handler.AssignRegionHandler(153): Opened testBlockEvictionOnRegionMove1732777004511,,1732777004558.9dd73ea1b712ae668b4c7cba5ad52740. 2024-11-28T06:56:46,924 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=9dd73ea1b712ae668b4c7cba5ad52740, regionState=OPEN, openSeqNum=10, regionLocation=1cb9cf437a36,35569,1732777001264 2024-11-28T06:56:46,928 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-4-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=9, ppid=7, state=RUNNABLE, hasLock=false; OpenRegionProcedure 9dd73ea1b712ae668b4c7cba5ad52740, server=1cb9cf437a36,35569,1732777001264 because future has completed 2024-11-28T06:56:46,932 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=9, resume processing ppid=7 2024-11-28T06:56:46,932 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=9, ppid=7, state=SUCCESS, hasLock=false; OpenRegionProcedure 9dd73ea1b712ae668b4c7cba5ad52740, server=1cb9cf437a36,35569,1732777001264 in 183 msec 2024-11-28T06:56:46,934 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=7, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1732777004511, region=9dd73ea1b712ae668b4c7cba5ad52740, REOPEN/MOVE in 539 msec 2024-11-28T06:56:46,947 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster 2024-11-28T06:56:46,948 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-28T06:56:46,948 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79) at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611) at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.io.hfile.TestBlockEvictionOnRegionMovement.tearDown(TestBlockEvictionOnRegionMovement.java:174) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-28T06:56:46,952 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T06:56:46,953 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T06:56:46,953 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-28T06:56:46,954 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2024-11-28T06:56:46,954 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=68825266, stopped=false 2024-11-28T06:56:46,954 INFO [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=1cb9cf437a36,38499,1732776999991 2024-11-28T06:56:46,986 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37673-0x10181844d810001, quorum=127.0.0.1:58909, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-28T06:56:46,986 DEBUG [pool-21-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35569-0x10181844d810002, quorum=127.0.0.1:58909, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-28T06:56:46,986 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37673-0x10181844d810001, quorum=127.0.0.1:58909, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-28T06:56:46,986 DEBUG [pool-21-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35569-0x10181844d810002, quorum=127.0.0.1:58909, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-28T06:56:46,986 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38499-0x10181844d810000, quorum=127.0.0.1:58909, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-28T06:56:46,986 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38499-0x10181844d810000, quorum=127.0.0.1:58909, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-28T06:56:46,986 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-28T06:56:46,987 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:37673-0x10181844d810001, quorum=127.0.0.1:58909, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-28T06:56:46,987 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-28T06:56:46,987 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277) at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265) at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416) at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.io.hfile.TestBlockEvictionOnRegionMovement.tearDown(TestBlockEvictionOnRegionMovement.java:174) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-28T06:56:46,987 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T06:56:46,987 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '1cb9cf437a36,37673,1732777001059' ***** 2024-11-28T06:56:46,988 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-11-28T06:56:46,988 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '1cb9cf437a36,35569,1732777001264' ***** 2024-11-28T06:56:46,988 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:35569-0x10181844d810002, quorum=127.0.0.1:58909, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-28T06:56:46,988 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-11-28T06:56:46,988 INFO [RS:1;1cb9cf437a36:35569 {}] bucket.BucketCache(1738): Disabling cache 2024-11-28T06:56:46,988 INFO [RS:0;1cb9cf437a36:37673 {}] bucket.BucketCache(1738): Disabling cache 2024-11-28T06:56:46,988 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:38499-0x10181844d810000, quorum=127.0.0.1:58909, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-28T06:56:46,989 INFO [RS:1;1cb9cf437a36:35569 {}] bucket.BucketCache(1762): Shutdown bucket cache: IO persistent=true; path to write=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7e9a7384-d065-9ff8-196a-59688d3a4920/bucket.persistence 2024-11-28T06:56:46,989 INFO [Time-limited test-BucketCacheWriter-0 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-0 exiting, cacheEnabled=false 2024-11-28T06:56:46,989 INFO [Time-limited test-BucketCacheWriter-1 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-1 exiting, cacheEnabled=false 2024-11-28T06:56:46,990 INFO [Time-limited test-BucketCacheWriter-2 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-2 exiting, cacheEnabled=false 2024-11-28T06:56:46,990 INFO [RS:0;1cb9cf437a36:37673 {}] bucket.BucketCache(1762): Shutdown bucket cache: IO persistent=true; path to write=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7e9a7384-d065-9ff8-196a-59688d3a4920/bucket.persistence 2024-11-28T06:56:46,990 INFO [Time-limited test-BucketCacheWriter-0 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-0 exiting, cacheEnabled=false 2024-11-28T06:56:46,990 INFO [Time-limited test-BucketCacheWriter-2 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-2 exiting, cacheEnabled=false 2024-11-28T06:56:46,990 INFO [Time-limited test-BucketCacheWriter-1 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-1 exiting, cacheEnabled=false 2024-11-28T06:56:46,990 INFO [RS:0;1cb9cf437a36:37673 {}] bucket.BucketCache(1768): Shutting down cache persister thread. 2024-11-28T06:56:46,990 INFO [RS:1;1cb9cf437a36:35569 {}] bucket.BucketCache(1768): Shutting down cache persister thread. 2024-11-28T06:56:46,990 WARN [bucket-cache-persister {}] bucket.BucketCachePersister(64): Interrupting BucketCachePersister thread. java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCachePersister.run(BucketCachePersister.java:45) ~[classes/:?] 2024-11-28T06:56:46,990 WARN [bucket-cache-persister {}] bucket.BucketCachePersister(64): Interrupting BucketCachePersister thread. java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCachePersister.run(BucketCachePersister.java:45) ~[classes/:?] 2024-11-28T06:56:47,000 DEBUG [RS:0;1cb9cf437a36:37673 {}] bucket.BucketCache(1487): Thread RS:0;1cb9cf437a36:37673 started persisting bucket cache to file 2024-11-28T06:56:47,000 DEBUG [RS:1;1cb9cf437a36:35569 {}] bucket.BucketCache(1487): Thread RS:1;1cb9cf437a36:35569 started persisting bucket cache to file 2024-11-28T06:56:47,000 DEBUG [RS:1;1cb9cf437a36:35569 {}] bucket.BucketCache(1494): Persist in new chunked persistence format. 2024-11-28T06:56:47,000 DEBUG [RS:0;1cb9cf437a36:37673 {}] bucket.BucketCache(1494): Persist in new chunked persistence format. 2024-11-28T06:56:47,000 DEBUG [RS:1;1cb9cf437a36:35569 {}] bucket.BucketCache(1673): persistToFile: before persisting backing map size: 0, fullycachedFiles size: 0, chunkSize: 10000 2024-11-28T06:56:47,000 DEBUG [RS:0;1cb9cf437a36:37673 {}] bucket.BucketCache(1673): persistToFile: before persisting backing map size: 0, fullycachedFiles size: 0, chunkSize: 10000 2024-11-28T06:56:47,005 DEBUG [RS:1;1cb9cf437a36:35569 {}] bucket.BucketCache(1680): persistToFile: after persisting backing map size: 0, fullycachedFiles size: 0 2024-11-28T06:56:47,005 DEBUG [RS:1;1cb9cf437a36:35569 {}] bucket.BucketCache(1498): PersistToFile: after persisting backing map size: 0, fullycachedFiles size: 0, file name: bucket.persistence1732777007000 2024-11-28T06:56:47,005 DEBUG [RS:0;1cb9cf437a36:37673 {}] bucket.BucketCache(1680): persistToFile: after persisting backing map size: 0, fullycachedFiles size: 0 2024-11-28T06:56:47,005 DEBUG [RS:0;1cb9cf437a36:37673 {}] bucket.BucketCache(1498): PersistToFile: after persisting backing map size: 0, fullycachedFiles size: 0, file name: bucket.persistence1732777007000 2024-11-28T06:56:47,005 DEBUG [RS:0;1cb9cf437a36:37673 {}] bucket.BucketCache(1509): Thread RS:0;1cb9cf437a36:37673 finished persisting bucket cache to file, renaming 2024-11-28T06:56:47,006 INFO [RS:0;1cb9cf437a36:37673 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-28T06:56:47,006 INFO [RS:0;1cb9cf437a36:37673 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-28T06:56:47,006 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-28T06:56:47,006 INFO [RS:0;1cb9cf437a36:37673 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-28T06:56:47,006 INFO [RS:0;1cb9cf437a36:37673 {}] regionserver.HRegionServer(959): stopping server 1cb9cf437a36,37673,1732777001059 2024-11-28T06:56:47,006 INFO [RS:0;1cb9cf437a36:37673 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-28T06:56:47,006 INFO [RS:0;1cb9cf437a36:37673 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;1cb9cf437a36:37673. 2024-11-28T06:56:47,006 DEBUG [RS:0;1cb9cf437a36:37673 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-28T06:56:47,006 DEBUG [RS:0;1cb9cf437a36:37673 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T06:56:47,007 INFO [RS:0;1cb9cf437a36:37673 {}] regionserver.HRegionServer(976): stopping server 1cb9cf437a36,37673,1732777001059; all regions closed. 2024-11-28T06:56:47,011 DEBUG [RS:0;1cb9cf437a36:37673 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7e9a7384-d065-9ff8-196a-59688d3a4920/752619e7-664e-6a55-3bd6-93cba9c9ba92/oldWALs 2024-11-28T06:56:47,011 INFO [RS:0;1cb9cf437a36:37673 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 1cb9cf437a36%2C37673%2C1732777001059:(num 1732777003656) 2024-11-28T06:56:47,011 DEBUG [RS:0;1cb9cf437a36:37673 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T06:56:47,011 INFO [RS:0;1cb9cf437a36:37673 {}] regionserver.LeaseManager(133): Closed leases 2024-11-28T06:56:47,011 INFO [RS:0;1cb9cf437a36:37673 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-28T06:56:47,011 INFO [RS:0;1cb9cf437a36:37673 {}] hbase.ChoreService(370): Chore service for: regionserver/1cb9cf437a36:0 had [ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS] on shutdown 2024-11-28T06:56:47,012 INFO [regionserver/1cb9cf437a36:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-28T06:56:47,012 INFO [RS:0;1cb9cf437a36:37673 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-28T06:56:47,012 INFO [RS:0;1cb9cf437a36:37673 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-28T06:56:47,012 INFO [RS:0;1cb9cf437a36:37673 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-28T06:56:47,012 INFO [RS:0;1cb9cf437a36:37673 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-28T06:56:47,012 INFO [RS:0;1cb9cf437a36:37673 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:37673 2024-11-28T06:56:47,021 DEBUG [RS:1;1cb9cf437a36:35569 {}] bucket.BucketCache(1509): Thread RS:1;1cb9cf437a36:35569 finished persisting bucket cache to file, renaming 2024-11-28T06:56:47,021 WARN [RS:1;1cb9cf437a36:35569 {}] bucket.BucketCache(1512): Failed to commit cache persistent file. We might lose cached blocks if RS crashes/restarts before we successfully checkpoint again. 2024-11-28T06:56:47,021 INFO [RS:1;1cb9cf437a36:35569 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-28T06:56:47,021 INFO [RS:1;1cb9cf437a36:35569 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-28T06:56:47,021 INFO [RS:1;1cb9cf437a36:35569 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-28T06:56:47,022 INFO [RS:1;1cb9cf437a36:35569 {}] regionserver.HRegionServer(3091): Received CLOSE for 9dd73ea1b712ae668b4c7cba5ad52740 2024-11-28T06:56:47,022 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-28T06:56:47,022 INFO [RS:1;1cb9cf437a36:35569 {}] regionserver.HRegionServer(959): stopping server 1cb9cf437a36,35569,1732777001264 2024-11-28T06:56:47,022 INFO [RS:1;1cb9cf437a36:35569 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-28T06:56:47,022 INFO [RS:1;1cb9cf437a36:35569 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:1;1cb9cf437a36:35569. 2024-11-28T06:56:47,022 DEBUG [RS:1;1cb9cf437a36:35569 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-28T06:56:47,022 DEBUG [RS:1;1cb9cf437a36:35569 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T06:56:47,022 INFO [RS_CLOSE_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1662): Closing region testBlockEvictionOnRegionMove1732777004511,,1732777004558.9dd73ea1b712ae668b4c7cba5ad52740. during a graceful stop, and cache persistence is on, so setting evict on close to false. 2024-11-28T06:56:47,023 INFO [RS:1;1cb9cf437a36:35569 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-28T06:56:47,023 INFO [RS:1;1cb9cf437a36:35569 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-28T06:56:47,023 INFO [RS:1;1cb9cf437a36:35569 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-28T06:56:47,023 INFO [RS:1;1cb9cf437a36:35569 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-11-28T06:56:47,023 DEBUG [RS_CLOSE_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 9dd73ea1b712ae668b4c7cba5ad52740, disabling compactions & flushes 2024-11-28T06:56:47,023 INFO [RS_CLOSE_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testBlockEvictionOnRegionMove1732777004511,,1732777004558.9dd73ea1b712ae668b4c7cba5ad52740. 2024-11-28T06:56:47,023 DEBUG [RS_CLOSE_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testBlockEvictionOnRegionMove1732777004511,,1732777004558.9dd73ea1b712ae668b4c7cba5ad52740. 2024-11-28T06:56:47,023 DEBUG [RS_CLOSE_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testBlockEvictionOnRegionMove1732777004511,,1732777004558.9dd73ea1b712ae668b4c7cba5ad52740. after waiting 0 ms 2024-11-28T06:56:47,023 DEBUG [RS_CLOSE_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testBlockEvictionOnRegionMove1732777004511,,1732777004558.9dd73ea1b712ae668b4c7cba5ad52740. 2024-11-28T06:56:47,023 INFO [RS:1;1cb9cf437a36:35569 {}] regionserver.HRegionServer(1321): Waiting on 2 regions to close 2024-11-28T06:56:47,023 INFO [RS_CLOSE_META-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1662): Closing region hbase:meta,,1.1588230740 during a graceful stop, and cache persistence is on, so setting evict on close to false. 2024-11-28T06:56:47,023 DEBUG [RS:1;1cb9cf437a36:35569 {}] regionserver.HRegionServer(1325): Online Regions={1588230740=hbase:meta,,1.1588230740, 9dd73ea1b712ae668b4c7cba5ad52740=testBlockEvictionOnRegionMove1732777004511,,1732777004558.9dd73ea1b712ae668b4c7cba5ad52740.} 2024-11-28T06:56:47,023 DEBUG [RS_CLOSE_META-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-28T06:56:47,023 INFO [RS_CLOSE_META-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-28T06:56:47,023 DEBUG [RS_CLOSE_META-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-28T06:56:47,023 DEBUG [RS_CLOSE_META-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-28T06:56:47,024 DEBUG [RS_CLOSE_META-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-28T06:56:47,024 DEBUG [RS:1;1cb9cf437a36:35569 {}] regionserver.HRegionServer(1351): Waiting on 1588230740, 9dd73ea1b712ae668b4c7cba5ad52740 2024-11-28T06:56:47,024 DEBUG [StoreFileCloser-9dd73ea1b712ae668b4c7cba5ad52740-family-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7e9a7384-d065-9ff8-196a-59688d3a4920/752619e7-664e-6a55-3bd6-93cba9c9ba92/data/default/testBlockEvictionOnRegionMove1732777004511/9dd73ea1b712ae668b4c7cba5ad52740/family/2a9a061c857d42db9c7923afadf27520 2024-11-28T06:56:47,024 INFO [RS_CLOSE_META-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=3.70 KB heapSize=6.72 KB 2024-11-28T06:56:47,029 DEBUG [RS_CLOSE_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7e9a7384-d065-9ff8-196a-59688d3a4920/752619e7-664e-6a55-3bd6-93cba9c9ba92/data/default/testBlockEvictionOnRegionMove1732777004511/9dd73ea1b712ae668b4c7cba5ad52740/recovered.edits/12.seqid, newMaxSeqId=12, maxSeqId=9 2024-11-28T06:56:47,030 INFO [RS_CLOSE_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testBlockEvictionOnRegionMove1732777004511,,1732777004558.9dd73ea1b712ae668b4c7cba5ad52740. 2024-11-28T06:56:47,030 DEBUG [RS_CLOSE_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 9dd73ea1b712ae668b4c7cba5ad52740: Waiting for close lock at 1732777007022Running coprocessor pre-close hooks at 1732777007023 (+1 ms)Disabling compacts and flushes for region at 1732777007023Disabling writes for close at 1732777007023Writing region close event to WAL at 1732777007024 (+1 ms)Running coprocessor post-close hooks at 1732777007030 (+6 ms)Closed at 1732777007030 2024-11-28T06:56:47,030 DEBUG [RS_CLOSE_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testBlockEvictionOnRegionMove1732777004511,,1732777004558.9dd73ea1b712ae668b4c7cba5ad52740. 2024-11-28T06:56:47,034 DEBUG [RS_CLOSE_META-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 18a85ebfd3ac406f9d25de6a5415a5e1 2024-11-28T06:56:47,044 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37673-0x10181844d810001, quorum=127.0.0.1:58909, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/1cb9cf437a36,37673,1732777001059 2024-11-28T06:56:47,044 INFO [RS:0;1cb9cf437a36:37673 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-28T06:56:47,044 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38499-0x10181844d810000, quorum=127.0.0.1:58909, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-28T06:56:47,057 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [1cb9cf437a36,37673,1732777001059] 2024-11-28T06:56:47,057 DEBUG [RS_CLOSE_META-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7e9a7384-d065-9ff8-196a-59688d3a4920/752619e7-664e-6a55-3bd6-93cba9c9ba92/data/hbase/meta/1588230740/.tmp/info/18a85ebfd3ac406f9d25de6a5415a5e1 is 205, key is testBlockEvictionOnRegionMove1732777004511,,1732777004558.9dd73ea1b712ae668b4c7cba5ad52740./info:regioninfo/1732777006924/Put/seqid=0 2024-11-28T06:56:47,058 INFO [RS_CLOSE_META-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=3.48 KB at sequenceid=15 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7e9a7384-d065-9ff8-196a-59688d3a4920/752619e7-664e-6a55-3bd6-93cba9c9ba92/data/hbase/meta/1588230740/.tmp/info/18a85ebfd3ac406f9d25de6a5415a5e1 2024-11-28T06:56:47,068 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/1cb9cf437a36,37673,1732777001059 already deleted, retry=false 2024-11-28T06:56:47,069 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; 1cb9cf437a36,37673,1732777001059 expired; onlineServers=1 2024-11-28T06:56:47,069 DEBUG [RS_CLOSE_META-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFile$WriterFactory(308): Unable to set drop behind on e01526d8bb9c44f5a01db626807cda24 2024-11-28T06:56:47,070 INFO [regionserver/1cb9cf437a36:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-28T06:56:47,072 INFO [regionserver/1cb9cf437a36:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-28T06:56:47,088 DEBUG [RS_CLOSE_META-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7e9a7384-d065-9ff8-196a-59688d3a4920/752619e7-664e-6a55-3bd6-93cba9c9ba92/data/hbase/meta/1588230740/.tmp/ns/e01526d8bb9c44f5a01db626807cda24 is 43, key is default/ns:d/1732777004275/Put/seqid=0 2024-11-28T06:56:47,089 INFO [RS_CLOSE_META-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=74 B at sequenceid=15 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7e9a7384-d065-9ff8-196a-59688d3a4920/752619e7-664e-6a55-3bd6-93cba9c9ba92/data/hbase/meta/1588230740/.tmp/ns/e01526d8bb9c44f5a01db626807cda24 2024-11-28T06:56:47,100 DEBUG [RS_CLOSE_META-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFile$WriterFactory(308): Unable to set drop behind on b1ceb53df3f043f1b29d74ae36fc953c 2024-11-28T06:56:47,119 DEBUG [RS_CLOSE_META-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7e9a7384-d065-9ff8-196a-59688d3a4920/752619e7-664e-6a55-3bd6-93cba9c9ba92/data/hbase/meta/1588230740/.tmp/table/b1ceb53df3f043f1b29d74ae36fc953c is 78, key is testBlockEvictionOnRegionMove1732777004511/table:state/1732777005011/Put/seqid=0 2024-11-28T06:56:47,120 INFO [RS_CLOSE_META-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=148 B at sequenceid=15 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7e9a7384-d065-9ff8-196a-59688d3a4920/752619e7-664e-6a55-3bd6-93cba9c9ba92/data/hbase/meta/1588230740/.tmp/table/b1ceb53df3f043f1b29d74ae36fc953c 2024-11-28T06:56:47,123 DEBUG [RS_CLOSE_META-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7e9a7384-d065-9ff8-196a-59688d3a4920/752619e7-664e-6a55-3bd6-93cba9c9ba92/data/hbase/meta/1588230740/.tmp/info/18a85ebfd3ac406f9d25de6a5415a5e1 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7e9a7384-d065-9ff8-196a-59688d3a4920/752619e7-664e-6a55-3bd6-93cba9c9ba92/data/hbase/meta/1588230740/info/18a85ebfd3ac406f9d25de6a5415a5e1 2024-11-28T06:56:47,125 DEBUG [RS_CLOSE_META-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_CLOSE_META}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7e9a7384-d065-9ff8-196a-59688d3a4920/752619e7-664e-6a55-3bd6-93cba9c9ba92/data/hbase/meta/1588230740/info/18a85ebfd3ac406f9d25de6a5415a5e1, delay=932 ms 2024-11-28T06:56:47,127 INFO [RS_CLOSE_META-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7e9a7384-d065-9ff8-196a-59688d3a4920/752619e7-664e-6a55-3bd6-93cba9c9ba92/data/hbase/meta/1588230740/info/18a85ebfd3ac406f9d25de6a5415a5e1, entries=15, sequenceid=15, filesize=7.6 K 2024-11-28T06:56:47,128 DEBUG [RS_CLOSE_META-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7e9a7384-d065-9ff8-196a-59688d3a4920/752619e7-664e-6a55-3bd6-93cba9c9ba92/data/hbase/meta/1588230740/.tmp/ns/e01526d8bb9c44f5a01db626807cda24 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7e9a7384-d065-9ff8-196a-59688d3a4920/752619e7-664e-6a55-3bd6-93cba9c9ba92/data/hbase/meta/1588230740/ns/e01526d8bb9c44f5a01db626807cda24 2024-11-28T06:56:47,129 DEBUG [RS_CLOSE_META-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_CLOSE_META}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7e9a7384-d065-9ff8-196a-59688d3a4920/752619e7-664e-6a55-3bd6-93cba9c9ba92/data/hbase/meta/1588230740/ns/e01526d8bb9c44f5a01db626807cda24, delay=937 ms 2024-11-28T06:56:47,131 INFO [RS_CLOSE_META-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7e9a7384-d065-9ff8-196a-59688d3a4920/752619e7-664e-6a55-3bd6-93cba9c9ba92/data/hbase/meta/1588230740/ns/e01526d8bb9c44f5a01db626807cda24, entries=2, sequenceid=15, filesize=5.0 K 2024-11-28T06:56:47,131 DEBUG [RS_CLOSE_META-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7e9a7384-d065-9ff8-196a-59688d3a4920/752619e7-664e-6a55-3bd6-93cba9c9ba92/data/hbase/meta/1588230740/.tmp/table/b1ceb53df3f043f1b29d74ae36fc953c as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7e9a7384-d065-9ff8-196a-59688d3a4920/752619e7-664e-6a55-3bd6-93cba9c9ba92/data/hbase/meta/1588230740/table/b1ceb53df3f043f1b29d74ae36fc953c 2024-11-28T06:56:47,133 DEBUG [RS_CLOSE_META-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_CLOSE_META}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7e9a7384-d065-9ff8-196a-59688d3a4920/752619e7-664e-6a55-3bd6-93cba9c9ba92/data/hbase/meta/1588230740/table/b1ceb53df3f043f1b29d74ae36fc953c, delay=985 ms 2024-11-28T06:56:47,135 INFO [RS_CLOSE_META-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7e9a7384-d065-9ff8-196a-59688d3a4920/752619e7-664e-6a55-3bd6-93cba9c9ba92/data/hbase/meta/1588230740/table/b1ceb53df3f043f1b29d74ae36fc953c, entries=2, sequenceid=15, filesize=5.3 K 2024-11-28T06:56:47,136 INFO [RS_CLOSE_META-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~3.70 KB/3789, heapSize ~6.42 KB/6576, currentSize=0 B/0 for 1588230740 in 111ms, sequenceid=15, compaction requested=false 2024-11-28T06:56:47,136 DEBUG [RS_CLOSE_META-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_CLOSE_META}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta' 2024-11-28T06:56:47,137 DEBUG [StoreFileCloser-1588230740-info-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7e9a7384-d065-9ff8-196a-59688d3a4920/752619e7-664e-6a55-3bd6-93cba9c9ba92/data/hbase/meta/1588230740/info/18a85ebfd3ac406f9d25de6a5415a5e1 2024-11-28T06:56:47,138 DEBUG [StoreFileCloser-1588230740-ns-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7e9a7384-d065-9ff8-196a-59688d3a4920/752619e7-664e-6a55-3bd6-93cba9c9ba92/data/hbase/meta/1588230740/ns/e01526d8bb9c44f5a01db626807cda24 2024-11-28T06:56:47,140 DEBUG [StoreFileCloser-1588230740-table-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7e9a7384-d065-9ff8-196a-59688d3a4920/752619e7-664e-6a55-3bd6-93cba9c9ba92/data/hbase/meta/1588230740/table/b1ceb53df3f043f1b29d74ae36fc953c 2024-11-28T06:56:47,149 DEBUG [RS_CLOSE_META-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7e9a7384-d065-9ff8-196a-59688d3a4920/752619e7-664e-6a55-3bd6-93cba9c9ba92/data/hbase/meta/1588230740/recovered.edits/18.seqid, newMaxSeqId=18, maxSeqId=1 2024-11-28T06:56:47,150 DEBUG [RS_CLOSE_META-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-28T06:56:47,150 INFO [RS_CLOSE_META-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-28T06:56:47,151 DEBUG [RS_CLOSE_META-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1732777007023Running coprocessor pre-close hooks at 1732777007023Disabling compacts and flushes for region at 1732777007023Disabling writes for close at 1732777007024 (+1 ms)Obtaining lock to block concurrent updates at 1732777007024Preparing flush snapshotting stores in 1588230740 at 1732777007024Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=3789, getHeapSize=6816, getOffHeapSize=0, getCellsCount=27 at 1732777007024Flushing stores of hbase:meta,,1.1588230740 at 1732777007025 (+1 ms)Flushing 1588230740/info: creating writer at 1732777007025Flushing 1588230740/info: appending metadata at 1732777007056 (+31 ms)Flushing 1588230740/info: closing flushed file at 1732777007056Flushing 1588230740/ns: creating writer at 1732777007062 (+6 ms)Flushing 1588230740/ns: appending metadata at 1732777007087 (+25 ms)Flushing 1588230740/ns: closing flushed file at 1732777007087Flushing 1588230740/table: creating writer at 1732777007092 (+5 ms)Flushing 1588230740/table: appending metadata at 1732777007118 (+26 ms)Flushing 1588230740/table: closing flushed file at 1732777007118Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@3e224f1e: reopening flushed file at 1732777007123 (+5 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@57eb6024: reopening flushed file at 1732777007127 (+4 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@2e4729c4: reopening flushed file at 1732777007131 (+4 ms)Finished flush of dataSize ~3.70 KB/3789, heapSize ~6.42 KB/6576, currentSize=0 B/0 for 1588230740 in 111ms, sequenceid=15, compaction requested=false at 1732777007136 (+5 ms)Writing region close event to WAL at 1732777007141 (+5 ms)Running coprocessor post-close hooks at 1732777007150 (+9 ms)Closed at 1732777007150 2024-11-28T06:56:47,151 DEBUG [RS_CLOSE_META-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-11-28T06:56:47,158 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37673-0x10181844d810001, quorum=127.0.0.1:58909, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-28T06:56:47,158 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37673-0x10181844d810001, quorum=127.0.0.1:58909, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-28T06:56:47,158 INFO [RS:0;1cb9cf437a36:37673 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-28T06:56:47,158 INFO [RS:0;1cb9cf437a36:37673 {}] regionserver.HRegionServer(1031): Exiting; stopping=1cb9cf437a36,37673,1732777001059; zookeeper connection closed. 2024-11-28T06:56:47,159 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@78f43c04 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@78f43c04 2024-11-28T06:56:47,224 INFO [RS:1;1cb9cf437a36:35569 {}] regionserver.HRegionServer(976): stopping server 1cb9cf437a36,35569,1732777001264; all regions closed. 2024-11-28T06:56:47,227 DEBUG [RS:1;1cb9cf437a36:35569 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7e9a7384-d065-9ff8-196a-59688d3a4920/752619e7-664e-6a55-3bd6-93cba9c9ba92/oldWALs 2024-11-28T06:56:47,227 INFO [RS:1;1cb9cf437a36:35569 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 1cb9cf437a36%2C35569%2C1732777001264.meta:.meta(num 1732777004103) 2024-11-28T06:56:47,229 DEBUG [RS:1;1cb9cf437a36:35569 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7e9a7384-d065-9ff8-196a-59688d3a4920/752619e7-664e-6a55-3bd6-93cba9c9ba92/oldWALs 2024-11-28T06:56:47,229 INFO [RS:1;1cb9cf437a36:35569 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 1cb9cf437a36%2C35569%2C1732777001264:(num 1732777003656) 2024-11-28T06:56:47,230 DEBUG [RS:1;1cb9cf437a36:35569 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T06:56:47,230 INFO [RS:1;1cb9cf437a36:35569 {}] regionserver.LeaseManager(133): Closed leases 2024-11-28T06:56:47,230 INFO [RS:1;1cb9cf437a36:35569 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-28T06:56:47,230 INFO [RS:1;1cb9cf437a36:35569 {}] hbase.ChoreService(370): Chore service for: regionserver/1cb9cf437a36:0 had [ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-11-28T06:56:47,230 INFO [RS:1;1cb9cf437a36:35569 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-28T06:56:47,230 INFO [regionserver/1cb9cf437a36:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-28T06:56:47,230 INFO [RS:1;1cb9cf437a36:35569 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:35569 2024-11-28T06:56:47,260 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38499-0x10181844d810000, quorum=127.0.0.1:58909, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-28T06:56:47,260 DEBUG [pool-21-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35569-0x10181844d810002, quorum=127.0.0.1:58909, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/1cb9cf437a36,35569,1732777001264 2024-11-28T06:56:47,261 INFO [RS:1;1cb9cf437a36:35569 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-28T06:56:47,268 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [1cb9cf437a36,35569,1732777001264] 2024-11-28T06:56:47,277 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/1cb9cf437a36,35569,1732777001264 already deleted, retry=false 2024-11-28T06:56:47,277 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; 1cb9cf437a36,35569,1732777001264 expired; onlineServers=0 2024-11-28T06:56:47,277 INFO [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master '1cb9cf437a36,38499,1732776999991' ***** 2024-11-28T06:56:47,277 INFO [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0 2024-11-28T06:56:47,277 INFO [M:0;1cb9cf437a36:38499 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-28T06:56:47,277 INFO [M:0;1cb9cf437a36:38499 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-28T06:56:47,278 DEBUG [M:0;1cb9cf437a36:38499 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2024-11-28T06:56:47,278 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2024-11-28T06:56:47,278 DEBUG [M:0;1cb9cf437a36:38499 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2024-11-28T06:56:47,278 DEBUG [master/1cb9cf437a36:0:becomeActiveMaster-HFileCleaner.large.0-1732777003228 {}] cleaner.HFileCleaner(306): Exit Thread[master/1cb9cf437a36:0:becomeActiveMaster-HFileCleaner.large.0-1732777003228,5,FailOnTimeoutGroup] 2024-11-28T06:56:47,278 INFO [M:0;1cb9cf437a36:38499 {}] hbase.ChoreService(370): Chore service for: master/1cb9cf437a36:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown 2024-11-28T06:56:47,278 INFO [M:0;1cb9cf437a36:38499 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-28T06:56:47,278 DEBUG [master/1cb9cf437a36:0:becomeActiveMaster-HFileCleaner.small.0-1732777003236 {}] cleaner.HFileCleaner(306): Exit Thread[master/1cb9cf437a36:0:becomeActiveMaster-HFileCleaner.small.0-1732777003236,5,FailOnTimeoutGroup] 2024-11-28T06:56:47,278 DEBUG [M:0;1cb9cf437a36:38499 {}] master.HMaster(1795): Stopping service threads 2024-11-28T06:56:47,278 INFO [M:0;1cb9cf437a36:38499 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2024-11-28T06:56:47,279 INFO [M:0;1cb9cf437a36:38499 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-28T06:56:47,279 INFO [M:0;1cb9cf437a36:38499 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-28T06:56:47,280 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2024-11-28T06:56:47,285 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38499-0x10181844d810000, quorum=127.0.0.1:58909, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-11-28T06:56:47,285 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38499-0x10181844d810000, quorum=127.0.0.1:58909, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-28T06:56:47,285 DEBUG [M:0;1cb9cf437a36:38499 {}] zookeeper.ZKUtil(347): master:38499-0x10181844d810000, quorum=127.0.0.1:58909, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error) 2024-11-28T06:56:47,286 WARN [M:0;1cb9cf437a36:38499 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null 2024-11-28T06:56:47,286 INFO [M:0;1cb9cf437a36:38499 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7e9a7384-d065-9ff8-196a-59688d3a4920/752619e7-664e-6a55-3bd6-93cba9c9ba92/.lastflushedseqids 2024-11-28T06:56:47,297 INFO [M:0;1cb9cf437a36:38499 {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-28T06:56:47,298 INFO [M:0;1cb9cf437a36:38499 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2024-11-28T06:56:47,298 DEBUG [M:0;1cb9cf437a36:38499 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-28T06:56:47,298 INFO [M:0;1cb9cf437a36:38499 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-28T06:56:47,298 DEBUG [M:0;1cb9cf437a36:38499 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-28T06:56:47,298 DEBUG [M:0;1cb9cf437a36:38499 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-28T06:56:47,298 DEBUG [M:0;1cb9cf437a36:38499 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-28T06:56:47,298 INFO [M:0;1cb9cf437a36:38499 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=34.67 KB heapSize=43.02 KB 2024-11-28T06:56:47,309 DEBUG [M:0;1cb9cf437a36:38499 {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on ddb4e54d077a4bd49d052cdfeece09e7 2024-11-28T06:56:47,323 DEBUG [M:0;1cb9cf437a36:38499 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7e9a7384-d065-9ff8-196a-59688d3a4920/752619e7-664e-6a55-3bd6-93cba9c9ba92/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/ddb4e54d077a4bd49d052cdfeece09e7 is 82, key is hbase:meta,,1/info:regioninfo/1732777004181/Put/seqid=0 2024-11-28T06:56:47,324 INFO [M:0;1cb9cf437a36:38499 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=504 B at sequenceid=84 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7e9a7384-d065-9ff8-196a-59688d3a4920/752619e7-664e-6a55-3bd6-93cba9c9ba92/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/ddb4e54d077a4bd49d052cdfeece09e7 2024-11-28T06:56:47,333 DEBUG [M:0;1cb9cf437a36:38499 {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on afd43029dc65405bbce8d105e9bbc5ec 2024-11-28T06:56:47,348 DEBUG [M:0;1cb9cf437a36:38499 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7e9a7384-d065-9ff8-196a-59688d3a4920/752619e7-664e-6a55-3bd6-93cba9c9ba92/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/afd43029dc65405bbce8d105e9bbc5ec is 778, key is \x00\x00\x00\x00\x00\x00\x00\x04/proc:d/1732777005018/Put/seqid=0 2024-11-28T06:56:47,349 INFO [M:0;1cb9cf437a36:38499 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=34.05 KB at sequenceid=84 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7e9a7384-d065-9ff8-196a-59688d3a4920/752619e7-664e-6a55-3bd6-93cba9c9ba92/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/afd43029dc65405bbce8d105e9bbc5ec 2024-11-28T06:56:47,361 DEBUG [M:0;1cb9cf437a36:38499 {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 53be802ef5744607a185b59f427a3507 2024-11-28T06:56:47,369 DEBUG [pool-21-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35569-0x10181844d810002, quorum=127.0.0.1:58909, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-28T06:56:47,369 INFO [RS:1;1cb9cf437a36:35569 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-28T06:56:47,369 DEBUG [pool-21-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35569-0x10181844d810002, quorum=127.0.0.1:58909, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-28T06:56:47,369 INFO [RS:1;1cb9cf437a36:35569 {}] regionserver.HRegionServer(1031): Exiting; stopping=1cb9cf437a36,35569,1732777001264; zookeeper connection closed. 2024-11-28T06:56:47,369 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@4fef772f {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@4fef772f 2024-11-28T06:56:47,370 INFO [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 2 regionserver(s) complete 2024-11-28T06:56:47,376 DEBUG [M:0;1cb9cf437a36:38499 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7e9a7384-d065-9ff8-196a-59688d3a4920/752619e7-664e-6a55-3bd6-93cba9c9ba92/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/53be802ef5744607a185b59f427a3507 is 69, key is 1cb9cf437a36,35569,1732777001264/rs:state/1732777003283/Put/seqid=0 2024-11-28T06:56:47,376 INFO [M:0;1cb9cf437a36:38499 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=130 B at sequenceid=84 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7e9a7384-d065-9ff8-196a-59688d3a4920/752619e7-664e-6a55-3bd6-93cba9c9ba92/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/53be802ef5744607a185b59f427a3507 2024-11-28T06:56:47,378 DEBUG [M:0;1cb9cf437a36:38499 {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7e9a7384-d065-9ff8-196a-59688d3a4920/752619e7-664e-6a55-3bd6-93cba9c9ba92/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/ddb4e54d077a4bd49d052cdfeece09e7 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7e9a7384-d065-9ff8-196a-59688d3a4920/752619e7-664e-6a55-3bd6-93cba9c9ba92/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/ddb4e54d077a4bd49d052cdfeece09e7 2024-11-28T06:56:47,381 INFO [M:0;1cb9cf437a36:38499 {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7e9a7384-d065-9ff8-196a-59688d3a4920/752619e7-664e-6a55-3bd6-93cba9c9ba92/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/ddb4e54d077a4bd49d052cdfeece09e7, entries=8, sequenceid=84, filesize=5.5 K 2024-11-28T06:56:47,381 DEBUG [M:0;1cb9cf437a36:38499 {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7e9a7384-d065-9ff8-196a-59688d3a4920/752619e7-664e-6a55-3bd6-93cba9c9ba92/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/afd43029dc65405bbce8d105e9bbc5ec as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7e9a7384-d065-9ff8-196a-59688d3a4920/752619e7-664e-6a55-3bd6-93cba9c9ba92/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/afd43029dc65405bbce8d105e9bbc5ec 2024-11-28T06:56:47,384 INFO [M:0;1cb9cf437a36:38499 {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7e9a7384-d065-9ff8-196a-59688d3a4920/752619e7-664e-6a55-3bd6-93cba9c9ba92/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/afd43029dc65405bbce8d105e9bbc5ec, entries=9, sequenceid=84, filesize=6.5 K 2024-11-28T06:56:47,384 DEBUG [M:0;1cb9cf437a36:38499 {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7e9a7384-d065-9ff8-196a-59688d3a4920/752619e7-664e-6a55-3bd6-93cba9c9ba92/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/53be802ef5744607a185b59f427a3507 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7e9a7384-d065-9ff8-196a-59688d3a4920/752619e7-664e-6a55-3bd6-93cba9c9ba92/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/53be802ef5744607a185b59f427a3507 2024-11-28T06:56:47,387 INFO [M:0;1cb9cf437a36:38499 {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7e9a7384-d065-9ff8-196a-59688d3a4920/752619e7-664e-6a55-3bd6-93cba9c9ba92/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/53be802ef5744607a185b59f427a3507, entries=2, sequenceid=84, filesize=5.1 K 2024-11-28T06:56:47,388 INFO [M:0;1cb9cf437a36:38499 {}] regionserver.HRegion(3140): Finished flush of dataSize ~34.67 KB/35506, heapSize ~42.73 KB/43752, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 90ms, sequenceid=84, compaction requested=false 2024-11-28T06:56:47,390 INFO [M:0;1cb9cf437a36:38499 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-28T06:56:47,391 DEBUG [M:0;1cb9cf437a36:38499 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1732777007298Disabling compacts and flushes for region at 1732777007298Disabling writes for close at 1732777007298Obtaining lock to block concurrent updates at 1732777007298Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1732777007298Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=35506, getHeapSize=43992, getOffHeapSize=0, getCellsCount=99 at 1732777007299 (+1 ms)Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1732777007299Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1732777007299Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1732777007323 (+24 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1732777007323Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1732777007326 (+3 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1732777007347 (+21 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1732777007347Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1732777007351 (+4 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1732777007375 (+24 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1732777007375Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@6fd99533: reopening flushed file at 1732777007378 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@4b06c1d6: reopening flushed file at 1732777007381 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@6d73914c: reopening flushed file at 1732777007384 (+3 ms)Finished flush of dataSize ~34.67 KB/35506, heapSize ~42.73 KB/43752, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 90ms, sequenceid=84, compaction requested=false at 1732777007388 (+4 ms)Writing region close event to WAL at 1732777007390 (+2 ms)Closed at 1732777007390 2024-11-28T06:56:47,394 INFO [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-28T06:56:47,394 INFO [M:0;1cb9cf437a36:38499 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down. 2024-11-28T06:56:47,395 INFO [M:0;1cb9cf437a36:38499 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:38499 2024-11-28T06:56:47,395 INFO [M:0;1cb9cf437a36:38499 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-28T06:56:47,511 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38499-0x10181844d810000, quorum=127.0.0.1:58909, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-28T06:56:47,511 INFO [M:0;1cb9cf437a36:38499 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-28T06:56:47,511 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38499-0x10181844d810000, quorum=127.0.0.1:58909, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-28T06:56:47,518 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers 2024-11-28T06:56:47,543 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down 2024-11-28T06:56:47,543 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-28T06:56:47,553 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: io.hfile.TestBlockEvictionOnRegionMovement#testBlockEvictionOnRegionMove Thread=52 (was 12) Potentially hanging thread: master/1cb9cf437a36:0:becomeActiveMaster-MemStoreChunkPool Statistics java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HBase-Metrics2-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: region-location-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1170) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-4-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-4-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: regionserver/1cb9cf437a36:0.procedureResultReporter java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.RemoteProcedureResultReporter.run(RemoteProcedureResultReporter.java:75) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: hfile-prefetch-1732777006907 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1177) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-3-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: org.apache.hadoop.fs.FileSystem$Statistics$StatisticsDataReferenceCleaner java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:155) java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:176) app//org.apache.hadoop.fs.FileSystem$Statistics$StatisticsDataReferenceCleaner.run(FileSystem.java:4171) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: hfile-prefetch-1732777007129 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1177) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HMaster-EventLoopGroup-1-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: SnapshotHandlerChoreCleaner java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-3-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-4-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: master/1cb9cf437a36:0:becomeActiveMaster-MemStoreChunkPool Statistics java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Idle-Rpc-Conn-Sweeper-pool-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: regionserver/1cb9cf437a36:0.procedureResultReporter java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.RemoteProcedureResultReporter.run(RemoteProcedureResultReporter.java:75) Potentially hanging thread: RpcClient-timer-pool-0 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.waitForNextTick(HashedWheelTimer.java:598) app//org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:494) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Timer for 'HBase' metrics system java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.util.TimerThread.mainLoop(Timer.java:563) java.base@17.0.11/java.util.TimerThread.run(Timer.java:516) Potentially hanging thread: hfile-prefetch-1732777007125 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1177) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HMaster-EventLoopGroup-1-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-4 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ForkJoinPool.commonPool-worker-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkUntil(LockSupport.java:410) java.base@17.0.11/java.util.concurrent.ForkJoinPool.awaitWork(ForkJoinPool.java:1726) java.base@17.0.11/java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1623) java.base@17.0.11/java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:165) Potentially hanging thread: ForkJoinPool.commonPool-worker-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.ForkJoinPool.awaitWork(ForkJoinPool.java:1724) java.base@17.0.11/java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1623) java.base@17.0.11/java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:165) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-3-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: region-location-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1170) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: SessionTracker java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.zookeeper.server.SessionTrackerImpl.run(SessionTrackerImpl.java:163) Potentially hanging thread: HMaster-EventLoopGroup-1-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Async-Client-Retry-Timer-pool-0 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.waitForNextTick(HashedWheelTimer.java:598) app//org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:494) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Time-limited test.named-queue-events-pool-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) app//com.lmax.disruptor.BlockingWaitStrategy.waitFor(BlockingWaitStrategy.java:47) app//com.lmax.disruptor.ProcessingSequenceBarrier.waitFor(ProcessingSequenceBarrier.java:56) app//com.lmax.disruptor.BatchEventProcessor.processEvents(BatchEventProcessor.java:159) app//com.lmax.disruptor.BatchEventProcessor.run(BatchEventProcessor.java:125) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: hfile-prefetch-1732777005366 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-5 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Monitor thread for TaskMonitor java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hbase.monitoring.TaskMonitor$MonitorRunnable.run(TaskMonitor.java:325) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ForkJoinPool.commonPool-worker-3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.ForkJoinPool.awaitWork(ForkJoinPool.java:1724) java.base@17.0.11/java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1623) java.base@17.0.11/java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:165) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ForkJoinPool.commonPool-worker-4 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.ForkJoinPool.awaitWork(ForkJoinPool.java:1724) java.base@17.0.11/java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1623) java.base@17.0.11/java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:165) Potentially hanging thread: weak-ref-cleaner-strictcontextstorage java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:155) java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:176) app//io.opentelemetry.context.StrictContextStorage$PendingScopes.run(StrictContextStorage.java:269) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=346 (was 287) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=368 (was 346) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=8129 (was 8483) 2024-11-28T06:56:47,561 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: io.hfile.TestBlockEvictionOnRegionMovement#testBlockEvictionOnGracefulStop Thread=52, OpenFileDescriptor=346, MaxFileDescriptor=1048576, SystemLoadAverage=368, ProcessCount=11, AvailableMemoryMB=8129 2024-11-28T06:56:47,561 INFO [Time-limited test {}] hbase.HBaseTestingUtil(401): System.getProperty("hadoop.log.dir") already set to: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7e9a7384-d065-9ff8-196a-59688d3a4920/hadoop-log-dir so I do NOT create it in target/test-data/33d1270a-7ecb-879e-376a-4fd885e9a050 2024-11-28T06:56:47,562 INFO [Time-limited test {}] hbase.HBaseTestingUtil(401): System.getProperty("hadoop.tmp.dir") already set to: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7e9a7384-d065-9ff8-196a-59688d3a4920/hadoop-tmp-dir so I do NOT create it in target/test-data/33d1270a-7ecb-879e-376a-4fd885e9a050 2024-11-28T06:56:47,562 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-28T06:56:47,564 INFO [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/33d1270a-7ecb-879e-376a-4fd885e9a050/cluster_b83fb8f6-eb71-1148-ca18-7c8a7e102aed, deleteOnExit=true 2024-11-28T06:56:47,571 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/33d1270a-7ecb-879e-376a-4fd885e9a050/cluster_b83fb8f6-eb71-1148-ca18-7c8a7e102aed/zookeeper_0, clientPort=61052, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/33d1270a-7ecb-879e-376a-4fd885e9a050/cluster_b83fb8f6-eb71-1148-ca18-7c8a7e102aed/zookeeper_0/version-2, dataDirSize=457 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/33d1270a-7ecb-879e-376a-4fd885e9a050/cluster_b83fb8f6-eb71-1148-ca18-7c8a7e102aed/zookeeper_0/version-2, dataLogSize=457 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0 2024-11-28T06:56:47,573 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=61052 2024-11-28T06:56:47,588 INFO [Time-limited test {}] util.FSUtils(489): Created version file at /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7e9a7384-d065-9ff8-196a-59688d3a4920/752619e7-664e-6a55-3bd6-93cba9c9ba92 with version=8 2024-11-28T06:56:47,588 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1141): The hbase.fs.tmp.dir is set to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7e9a7384-d065-9ff8-196a-59688d3a4920/752619e7-664e-6a55-3bd6-93cba9c9ba92/hbase-staging 2024-11-28T06:56:47,591 INFO [Time-limited test {}] client.ConnectionUtils(128): master/1cb9cf437a36:0 server-side Connection retries=45 2024-11-28T06:56:47,592 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-28T06:56:47,592 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-28T06:56:47,592 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-28T06:56:47,592 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-28T06:56:47,592 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-28T06:56:47,592 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService 2024-11-28T06:56:47,592 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-28T06:56:47,593 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:42945 2024-11-28T06:56:47,595 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:42945 connecting to ZooKeeper ensemble=127.0.0.1:61052 2024-11-28T06:56:47,648 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:429450x0, quorum=127.0.0.1:61052, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-28T06:56:47,648 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:42945-0x10181846c940000 connected 2024-11-28T06:56:47,725 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-28T06:56:47,725 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-28T06:56:47,729 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:42945-0x10181846c940000, quorum=127.0.0.1:61052, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-28T06:56:47,730 INFO [Time-limited test {}] master.HMaster(525): hbase.rootdir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7e9a7384-d065-9ff8-196a-59688d3a4920/752619e7-664e-6a55-3bd6-93cba9c9ba92, hbase.cluster.distributed=false 2024-11-28T06:56:47,733 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:42945-0x10181846c940000, quorum=127.0.0.1:61052, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-28T06:56:47,734 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=42945 2024-11-28T06:56:47,734 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=42945 2024-11-28T06:56:47,734 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=42945 2024-11-28T06:56:47,736 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=42945 2024-11-28T06:56:47,738 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=42945 2024-11-28T06:56:47,755 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/1cb9cf437a36:0 server-side Connection retries=45 2024-11-28T06:56:47,756 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-28T06:56:47,756 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-28T06:56:47,756 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-28T06:56:47,756 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-28T06:56:47,756 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-28T06:56:47,756 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-28T06:56:47,756 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-28T06:56:47,757 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:40661 2024-11-28T06:56:47,759 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:40661 connecting to ZooKeeper ensemble=127.0.0.1:61052 2024-11-28T06:56:47,759 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-28T06:56:47,759 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-28T06:56:47,772 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:406610x0, quorum=127.0.0.1:61052, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-28T06:56:47,773 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:40661-0x10181846c940001 connected 2024-11-28T06:56:47,773 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:40661-0x10181846c940001, quorum=127.0.0.1:61052, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-28T06:56:47,773 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-28T06:56:47,774 INFO [Time-limited test {}] bucket.FileIOEngine(99): Allocating cache 400 MB, on the path:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/33d1270a-7ecb-879e-376a-4fd885e9a050/bucket.cache 2024-11-28T06:56:47,774 INFO [Time-limited test {}] bucket.BucketCache(364): Instantiating BucketCache with acceptableFactor: 0.95, minFactor: 0.85, extraFreeFactor: 0.1, singleFactor: 0.25, multiFactor: 0.5, memoryFactor: 0.25, useStrongRef: false 2024-11-28T06:56:47,774 INFO [Time-limited test {}] bucket.BucketCache(478): Starting BucketCachePersisterThread 2024-11-28T06:56:47,774 INFO [Time-limited test {}] bucket.BucketCachePersister(38): BucketCachePersister started with interval: 100 2024-11-28T06:56:47,775 INFO [Thread-119 {}] bucket.BucketCache(1530): Started retrieving bucket cache from file 2024-11-28T06:56:47,775 WARN [Thread-119 {}] bucket.BucketCache(1533): Persistence file missing! It's ok if it's first run after enabling persistent cache. 2024-11-28T06:56:47,775 INFO [Time-limited test {}] bucket.BucketCache(403): Started bucket cache; ioengine=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/33d1270a-7ecb-879e-376a-4fd885e9a050/bucket.cache, capacity=400 MB, blockSize=64 KB, writerThreadNum=3, writerQLen=64, persistencePath=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/33d1270a-7ecb-879e-376a-4fd885e9a050/bucket.persistence, bucketAllocator=org.apache.hadoop.hbase.io.hfile.bucket.BucketAllocator 2024-11-28T06:56:47,775 INFO [Thread-119 {}] bucket.BucketAllocator(343): Cache totalSize=418148352, buckets=199, bucket capacity=2101248=(4*525312)=(FEWEST_ITEMS_IN_BUCKET*(largest configured bucketcache size)) 2024-11-28T06:56:47,775 INFO [Thread-119 {}] bucket.BucketCache(413): Persistent bucket cache recovery from /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/33d1270a-7ecb-879e-376a-4fd885e9a050/bucket.persistence is complete. 2024-11-28T06:56:47,776 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-28T06:56:47,778 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:40661-0x10181846c940001, quorum=127.0.0.1:61052, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-28T06:56:47,780 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:40661-0x10181846c940001, quorum=127.0.0.1:61052, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-28T06:56:47,783 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=40661 2024-11-28T06:56:47,783 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=40661 2024-11-28T06:56:47,784 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=40661 2024-11-28T06:56:47,786 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=40661 2024-11-28T06:56:47,788 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=40661 2024-11-28T06:56:47,812 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/1cb9cf437a36:0 server-side Connection retries=45 2024-11-28T06:56:47,812 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-28T06:56:47,812 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-28T06:56:47,812 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-28T06:56:47,812 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-28T06:56:47,812 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-28T06:56:47,812 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-28T06:56:47,813 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-28T06:56:47,813 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:46291 2024-11-28T06:56:47,816 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:46291 connecting to ZooKeeper ensemble=127.0.0.1:61052 2024-11-28T06:56:47,816 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-28T06:56:47,816 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-28T06:56:47,851 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:462910x0, quorum=127.0.0.1:61052, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-28T06:56:47,851 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:462910x0, quorum=127.0.0.1:61052, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-28T06:56:47,852 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:46291-0x10181846c940002 connected 2024-11-28T06:56:47,852 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-28T06:56:47,853 INFO [Time-limited test {}] bucket.FileIOEngine(99): Allocating cache 400 MB, on the path:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/33d1270a-7ecb-879e-376a-4fd885e9a050/bucket.cache 2024-11-28T06:56:47,853 INFO [Time-limited test {}] bucket.BucketCache(364): Instantiating BucketCache with acceptableFactor: 0.95, minFactor: 0.85, extraFreeFactor: 0.1, singleFactor: 0.25, multiFactor: 0.5, memoryFactor: 0.25, useStrongRef: false 2024-11-28T06:56:47,853 INFO [Time-limited test {}] bucket.BucketCache(478): Starting BucketCachePersisterThread 2024-11-28T06:56:47,853 INFO [Time-limited test {}] bucket.BucketCachePersister(38): BucketCachePersister started with interval: 100 2024-11-28T06:56:47,855 INFO [Thread-120 {}] bucket.BucketCache(1530): Started retrieving bucket cache from file 2024-11-28T06:56:47,855 WARN [Thread-120 {}] bucket.BucketCache(1533): Persistence file missing! It's ok if it's first run after enabling persistent cache. 2024-11-28T06:56:47,855 INFO [Time-limited test {}] bucket.BucketCache(403): Started bucket cache; ioengine=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/33d1270a-7ecb-879e-376a-4fd885e9a050/bucket.cache, capacity=400 MB, blockSize=64 KB, writerThreadNum=3, writerQLen=64, persistencePath=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/33d1270a-7ecb-879e-376a-4fd885e9a050/bucket.persistence, bucketAllocator=org.apache.hadoop.hbase.io.hfile.bucket.BucketAllocator 2024-11-28T06:56:47,856 INFO [Thread-120 {}] bucket.BucketAllocator(343): Cache totalSize=418148352, buckets=199, bucket capacity=2101248=(4*525312)=(FEWEST_ITEMS_IN_BUCKET*(largest configured bucketcache size)) 2024-11-28T06:56:47,856 INFO [Thread-120 {}] bucket.BucketCache(413): Persistent bucket cache recovery from /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/33d1270a-7ecb-879e-376a-4fd885e9a050/bucket.persistence is complete. 2024-11-28T06:56:47,856 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-28T06:56:47,859 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:46291-0x10181846c940002, quorum=127.0.0.1:61052, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-28T06:56:47,863 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:46291-0x10181846c940002, quorum=127.0.0.1:61052, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-28T06:56:47,864 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=46291 2024-11-28T06:56:47,864 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=46291 2024-11-28T06:56:47,865 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=46291 2024-11-28T06:56:47,865 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=46291 2024-11-28T06:56:47,865 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=46291 2024-11-28T06:56:47,883 DEBUG [M:0;1cb9cf437a36:42945 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;1cb9cf437a36:42945 2024-11-28T06:56:47,883 INFO [master/1cb9cf437a36:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/1cb9cf437a36,42945,1732777007591 2024-11-28T06:56:47,892 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46291-0x10181846c940002, quorum=127.0.0.1:61052, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-28T06:56:47,892 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40661-0x10181846c940001, quorum=127.0.0.1:61052, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-28T06:56:47,892 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42945-0x10181846c940000, quorum=127.0.0.1:61052, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-28T06:56:47,894 DEBUG [master/1cb9cf437a36:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:42945-0x10181846c940000, quorum=127.0.0.1:61052, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/1cb9cf437a36,42945,1732777007591 2024-11-28T06:56:47,905 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46291-0x10181846c940002, quorum=127.0.0.1:61052, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-28T06:56:47,905 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42945-0x10181846c940000, quorum=127.0.0.1:61052, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-28T06:56:47,905 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46291-0x10181846c940002, quorum=127.0.0.1:61052, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-28T06:56:47,905 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40661-0x10181846c940001, quorum=127.0.0.1:61052, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-28T06:56:47,905 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40661-0x10181846c940001, quorum=127.0.0.1:61052, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-28T06:56:47,906 DEBUG [master/1cb9cf437a36:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:42945-0x10181846c940000, quorum=127.0.0.1:61052, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-11-28T06:56:47,908 INFO [master/1cb9cf437a36:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/1cb9cf437a36,42945,1732777007591 from backup master directory 2024-11-28T06:56:47,917 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46291-0x10181846c940002, quorum=127.0.0.1:61052, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-28T06:56:47,917 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42945-0x10181846c940000, quorum=127.0.0.1:61052, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/1cb9cf437a36,42945,1732777007591 2024-11-28T06:56:47,917 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40661-0x10181846c940001, quorum=127.0.0.1:61052, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-28T06:56:47,917 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42945-0x10181846c940000, quorum=127.0.0.1:61052, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-28T06:56:47,917 WARN [master/1cb9cf437a36:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-28T06:56:47,917 INFO [master/1cb9cf437a36:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=1cb9cf437a36,42945,1732777007591 2024-11-28T06:56:47,919 DEBUG [master/1cb9cf437a36:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7e9a7384-d065-9ff8-196a-59688d3a4920/752619e7-664e-6a55-3bd6-93cba9c9ba92/hbase.id] with ID: f1e003c1-57de-44c5-9f50-31707ec95057 2024-11-28T06:56:47,919 DEBUG [master/1cb9cf437a36:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7e9a7384-d065-9ff8-196a-59688d3a4920/752619e7-664e-6a55-3bd6-93cba9c9ba92/.tmp/hbase.id 2024-11-28T06:56:47,923 DEBUG [master/1cb9cf437a36:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7e9a7384-d065-9ff8-196a-59688d3a4920/752619e7-664e-6a55-3bd6-93cba9c9ba92/.tmp/hbase.id]:[file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7e9a7384-d065-9ff8-196a-59688d3a4920/752619e7-664e-6a55-3bd6-93cba9c9ba92/hbase.id] 2024-11-28T06:56:47,994 DEBUG [master/1cb9cf437a36:0:becomeActiveMaster {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-28T06:56:47,994 INFO [master/1cb9cf437a36:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem. 2024-11-28T06:56:47,997 INFO [master/1cb9cf437a36:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 3ms. 2024-11-28T06:56:48,017 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46291-0x10181846c940002, quorum=127.0.0.1:61052, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-28T06:56:48,017 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40661-0x10181846c940001, quorum=127.0.0.1:61052, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-28T06:56:48,017 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42945-0x10181846c940000, quorum=127.0.0.1:61052, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-28T06:56:48,022 INFO [master/1cb9cf437a36:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-28T06:56:48,023 INFO [master/1cb9cf437a36:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-11-28T06:56:48,024 INFO [master/1cb9cf437a36:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-28T06:56:48,045 INFO [master/1cb9cf437a36:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7e9a7384-d065-9ff8-196a-59688d3a4920/752619e7-664e-6a55-3bd6-93cba9c9ba92/MasterData/data/master/store 2024-11-28T06:56:48,052 DEBUG [master/1cb9cf437a36:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-28T06:56:48,052 DEBUG [master/1cb9cf437a36:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-28T06:56:48,052 INFO [master/1cb9cf437a36:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-28T06:56:48,052 DEBUG [master/1cb9cf437a36:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-28T06:56:48,052 DEBUG [master/1cb9cf437a36:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-28T06:56:48,052 DEBUG [master/1cb9cf437a36:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-28T06:56:48,052 INFO [master/1cb9cf437a36:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-28T06:56:48,052 DEBUG [master/1cb9cf437a36:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1732777008052Disabling compacts and flushes for region at 1732777008052Disabling writes for close at 1732777008052Writing region close event to WAL at 1732777008052Closed at 1732777008052 2024-11-28T06:56:48,053 WARN [master/1cb9cf437a36:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7e9a7384-d065-9ff8-196a-59688d3a4920/752619e7-664e-6a55-3bd6-93cba9c9ba92/MasterData/data/master/store/.initializing 2024-11-28T06:56:48,053 DEBUG [master/1cb9cf437a36:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7e9a7384-d065-9ff8-196a-59688d3a4920/752619e7-664e-6a55-3bd6-93cba9c9ba92/MasterData/WALs/1cb9cf437a36,42945,1732777007591 2024-11-28T06:56:48,063 INFO [master/1cb9cf437a36:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-28T06:56:48,067 INFO [master/1cb9cf437a36:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=1cb9cf437a36%2C42945%2C1732777007591, suffix=, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7e9a7384-d065-9ff8-196a-59688d3a4920/752619e7-664e-6a55-3bd6-93cba9c9ba92/MasterData/WALs/1cb9cf437a36,42945,1732777007591, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7e9a7384-d065-9ff8-196a-59688d3a4920/752619e7-664e-6a55-3bd6-93cba9c9ba92/MasterData/oldWALs, maxLogs=10 2024-11-28T06:56:48,111 INFO [master/1cb9cf437a36:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7e9a7384-d065-9ff8-196a-59688d3a4920/752619e7-664e-6a55-3bd6-93cba9c9ba92/MasterData/WALs/1cb9cf437a36,42945,1732777007591/1cb9cf437a36%2C42945%2C1732777007591.1732777008067 2024-11-28T06:56:48,113 DEBUG [master/1cb9cf437a36:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-11-28T06:56:48,113 DEBUG [master/1cb9cf437a36:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-11-28T06:56:48,113 DEBUG [master/1cb9cf437a36:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-28T06:56:48,113 DEBUG [master/1cb9cf437a36:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-11-28T06:56:48,113 DEBUG [master/1cb9cf437a36:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-11-28T06:56:48,115 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-11-28T06:56:48,124 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-11-28T06:56:48,124 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:56:48,125 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-28T06:56:48,125 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-11-28T06:56:48,130 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-11-28T06:56:48,130 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:56:48,130 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-28T06:56:48,131 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-11-28T06:56:48,134 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-11-28T06:56:48,134 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:56:48,134 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-28T06:56:48,135 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-11-28T06:56:48,138 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-11-28T06:56:48,138 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:56:48,138 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-28T06:56:48,139 DEBUG [master/1cb9cf437a36:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682 2024-11-28T06:56:48,139 DEBUG [master/1cb9cf437a36:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7e9a7384-d065-9ff8-196a-59688d3a4920/752619e7-664e-6a55-3bd6-93cba9c9ba92/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-28T06:56:48,139 DEBUG [master/1cb9cf437a36:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7e9a7384-d065-9ff8-196a-59688d3a4920/752619e7-664e-6a55-3bd6-93cba9c9ba92/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-28T06:56:48,139 DEBUG [master/1cb9cf437a36:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682 2024-11-28T06:56:48,139 DEBUG [master/1cb9cf437a36:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682 2024-11-28T06:56:48,140 DEBUG [master/1cb9cf437a36:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-28T06:56:48,140 DEBUG [master/1cb9cf437a36:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-11-28T06:56:48,146 DEBUG [master/1cb9cf437a36:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7e9a7384-d065-9ff8-196a-59688d3a4920/752619e7-664e-6a55-3bd6-93cba9c9ba92/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-28T06:56:48,147 INFO [master/1cb9cf437a36:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=70013706, jitterRate=0.043285518884658813}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-28T06:56:48,148 DEBUG [master/1cb9cf437a36:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682: Writing region info on filesystem at 1732777008113Initializing all the Stores at 1732777008114 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732777008114Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732777008115 (+1 ms)Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732777008115Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732777008115Cleaning up temporary data from old regions at 1732777008139 (+24 ms)Region opened successfully at 1732777008148 (+9 ms) 2024-11-28T06:56:48,149 INFO [master/1cb9cf437a36:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-11-28T06:56:48,153 DEBUG [master/1cb9cf437a36:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2a3fa147, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=1cb9cf437a36/172.17.0.2:0 2024-11-28T06:56:48,155 INFO [master/1cb9cf437a36:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating... 2024-11-28T06:56:48,155 INFO [master/1cb9cf437a36:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5 2024-11-28T06:56:48,155 INFO [master/1cb9cf437a36:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50 2024-11-28T06:56:48,155 INFO [master/1cb9cf437a36:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-28T06:56:48,156 INFO [master/1cb9cf437a36:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-28T06:56:48,156 INFO [master/1cb9cf437a36:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 0 msec 2024-11-28T06:56:48,156 INFO [master/1cb9cf437a36:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-11-28T06:56:48,159 INFO [master/1cb9cf437a36:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2024-11-28T06:56:48,160 DEBUG [master/1cb9cf437a36:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:42945-0x10181846c940000, quorum=127.0.0.1:61052, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error) 2024-11-28T06:56:48,183 DEBUG [master/1cb9cf437a36:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false 2024-11-28T06:56:48,184 INFO [master/1cb9cf437a36:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2024-11-28T06:56:48,185 DEBUG [master/1cb9cf437a36:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:42945-0x10181846c940000, quorum=127.0.0.1:61052, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error) 2024-11-28T06:56:48,191 DEBUG [master/1cb9cf437a36:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false 2024-11-28T06:56:48,192 INFO [master/1cb9cf437a36:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2024-11-28T06:56:48,193 DEBUG [master/1cb9cf437a36:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:42945-0x10181846c940000, quorum=127.0.0.1:61052, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error) 2024-11-28T06:56:48,200 DEBUG [master/1cb9cf437a36:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false 2024-11-28T06:56:48,201 DEBUG [master/1cb9cf437a36:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:42945-0x10181846c940000, quorum=127.0.0.1:61052, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error) 2024-11-28T06:56:48,208 DEBUG [master/1cb9cf437a36:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false 2024-11-28T06:56:48,211 DEBUG [master/1cb9cf437a36:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:42945-0x10181846c940000, quorum=127.0.0.1:61052, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error) 2024-11-28T06:56:48,221 DEBUG [master/1cb9cf437a36:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false 2024-11-28T06:56:48,233 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40661-0x10181846c940001, quorum=127.0.0.1:61052, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-28T06:56:48,233 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46291-0x10181846c940002, quorum=127.0.0.1:61052, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-28T06:56:48,233 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42945-0x10181846c940000, quorum=127.0.0.1:61052, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-28T06:56:48,233 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46291-0x10181846c940002, quorum=127.0.0.1:61052, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-28T06:56:48,233 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40661-0x10181846c940001, quorum=127.0.0.1:61052, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-28T06:56:48,234 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42945-0x10181846c940000, quorum=127.0.0.1:61052, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-28T06:56:48,235 INFO [master/1cb9cf437a36:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=1cb9cf437a36,42945,1732777007591, sessionid=0x10181846c940000, setting cluster-up flag (Was=false) 2024-11-28T06:56:48,255 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40661-0x10181846c940001, quorum=127.0.0.1:61052, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-28T06:56:48,255 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46291-0x10181846c940002, quorum=127.0.0.1:61052, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-28T06:56:48,255 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42945-0x10181846c940000, quorum=127.0.0.1:61052, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-28T06:56:48,283 DEBUG [master/1cb9cf437a36:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort 2024-11-28T06:56:48,285 DEBUG [master/1cb9cf437a36:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=1cb9cf437a36,42945,1732777007591 2024-11-28T06:56:48,305 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40661-0x10181846c940001, quorum=127.0.0.1:61052, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-28T06:56:48,305 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46291-0x10181846c940002, quorum=127.0.0.1:61052, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-28T06:56:48,305 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42945-0x10181846c940000, quorum=127.0.0.1:61052, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-28T06:56:48,330 DEBUG [master/1cb9cf437a36:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort 2024-11-28T06:56:48,331 DEBUG [master/1cb9cf437a36:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=1cb9cf437a36,42945,1732777007591 2024-11-28T06:56:48,338 INFO [master/1cb9cf437a36:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7e9a7384-d065-9ff8-196a-59688d3a4920/752619e7-664e-6a55-3bd6-93cba9c9ba92/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again 2024-11-28T06:56:48,340 DEBUG [master/1cb9cf437a36:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-28T06:56:48,341 INFO [master/1cb9cf437a36:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2 2024-11-28T06:56:48,341 INFO [master/1cb9cf437a36:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2024-11-28T06:56:48,341 DEBUG [master/1cb9cf437a36:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: 1cb9cf437a36,42945,1732777007591 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2024-11-28T06:56:48,343 DEBUG [master/1cb9cf437a36:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/1cb9cf437a36:0, corePoolSize=5, maxPoolSize=5 2024-11-28T06:56:48,343 DEBUG [master/1cb9cf437a36:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/1cb9cf437a36:0, corePoolSize=5, maxPoolSize=5 2024-11-28T06:56:48,343 DEBUG [master/1cb9cf437a36:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/1cb9cf437a36:0, corePoolSize=5, maxPoolSize=5 2024-11-28T06:56:48,343 DEBUG [master/1cb9cf437a36:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/1cb9cf437a36:0, corePoolSize=5, maxPoolSize=5 2024-11-28T06:56:48,343 DEBUG [master/1cb9cf437a36:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/1cb9cf437a36:0, corePoolSize=10, maxPoolSize=10 2024-11-28T06:56:48,343 DEBUG [master/1cb9cf437a36:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/1cb9cf437a36:0, corePoolSize=1, maxPoolSize=1 2024-11-28T06:56:48,343 DEBUG [master/1cb9cf437a36:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/1cb9cf437a36:0, corePoolSize=2, maxPoolSize=2 2024-11-28T06:56:48,344 DEBUG [master/1cb9cf437a36:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/1cb9cf437a36:0, corePoolSize=1, maxPoolSize=1 2024-11-28T06:56:48,348 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-28T06:56:48,348 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region 2024-11-28T06:56:48,349 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:56:48,349 INFO [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-11-28T06:56:48,363 INFO [master/1cb9cf437a36:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732777038363 2024-11-28T06:56:48,364 INFO [master/1cb9cf437a36:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2024-11-28T06:56:48,364 INFO [master/1cb9cf437a36:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2024-11-28T06:56:48,364 INFO [master/1cb9cf437a36:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2024-11-28T06:56:48,364 INFO [master/1cb9cf437a36:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2024-11-28T06:56:48,364 INFO [master/1cb9cf437a36:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2024-11-28T06:56:48,364 INFO [master/1cb9cf437a36:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2024-11-28T06:56:48,365 INFO [master/1cb9cf437a36:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-28T06:56:48,366 INFO [master/1cb9cf437a36:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2024-11-28T06:56:48,366 INFO [master/1cb9cf437a36:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2024-11-28T06:56:48,366 INFO [master/1cb9cf437a36:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2024-11-28T06:56:48,370 INFO [RS:0;1cb9cf437a36:40661 {}] regionserver.HRegionServer(746): ClusterId : f1e003c1-57de-44c5-9f50-31707ec95057 2024-11-28T06:56:48,370 DEBUG [RS:0;1cb9cf437a36:40661 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-28T06:56:48,370 INFO [RS:1;1cb9cf437a36:46291 {}] regionserver.HRegionServer(746): ClusterId : f1e003c1-57de-44c5-9f50-31707ec95057 2024-11-28T06:56:48,370 DEBUG [RS:1;1cb9cf437a36:46291 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-28T06:56:48,374 INFO [master/1cb9cf437a36:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2024-11-28T06:56:48,375 INFO [master/1cb9cf437a36:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2024-11-28T06:56:48,375 DEBUG [master/1cb9cf437a36:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/1cb9cf437a36:0:becomeActiveMaster-HFileCleaner.large.0-1732777008375,5,FailOnTimeoutGroup] 2024-11-28T06:56:48,378 DEBUG [master/1cb9cf437a36:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/1cb9cf437a36:0:becomeActiveMaster-HFileCleaner.small.0-1732777008375,5,FailOnTimeoutGroup] 2024-11-28T06:56:48,378 INFO [master/1cb9cf437a36:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-28T06:56:48,379 INFO [master/1cb9cf437a36:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2024-11-28T06:56:48,379 INFO [master/1cb9cf437a36:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2024-11-28T06:56:48,379 INFO [master/1cb9cf437a36:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2024-11-28T06:56:48,389 DEBUG [RS:0;1cb9cf437a36:40661 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-28T06:56:48,389 DEBUG [RS:0;1cb9cf437a36:40661 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-28T06:56:48,390 DEBUG [RS:1;1cb9cf437a36:46291 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-28T06:56:48,390 DEBUG [RS:1;1cb9cf437a36:46291 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-28T06:56:48,390 INFO [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7e9a7384-d065-9ff8-196a-59688d3a4920/752619e7-664e-6a55-3bd6-93cba9c9ba92/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321 2024-11-28T06:56:48,391 INFO [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7e9a7384-d065-9ff8-196a-59688d3a4920/752619e7-664e-6a55-3bd6-93cba9c9ba92 2024-11-28T06:56:48,402 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-28T06:56:48,406 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family info of region 1588230740 2024-11-28T06:56:48,410 DEBUG [RS:0;1cb9cf437a36:40661 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-28T06:56:48,410 DEBUG [RS:1;1cb9cf437a36:46291 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-28T06:56:48,410 DEBUG [RS:0;1cb9cf437a36:40661 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@8e2dc28, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=1cb9cf437a36/172.17.0.2:0 2024-11-28T06:56:48,410 DEBUG [RS:1;1cb9cf437a36:46291 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6a738f98, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=1cb9cf437a36/172.17.0.2:0 2024-11-28T06:56:48,413 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-28T06:56:48,413 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:56:48,414 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-28T06:56:48,414 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family ns of region 1588230740 2024-11-28T06:56:48,436 DEBUG [RS:0;1cb9cf437a36:40661 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;1cb9cf437a36:40661 2024-11-28T06:56:48,436 INFO [RS:0;1cb9cf437a36:40661 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-28T06:56:48,436 INFO [RS:0;1cb9cf437a36:40661 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-28T06:56:48,436 DEBUG [RS:0;1cb9cf437a36:40661 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-28T06:56:48,437 INFO [RS:0;1cb9cf437a36:40661 {}] regionserver.HRegionServer(2659): reportForDuty to master=1cb9cf437a36,42945,1732777007591 with port=40661, startcode=1732777007755 2024-11-28T06:56:48,438 DEBUG [RS:0;1cb9cf437a36:40661 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-28T06:56:48,439 DEBUG [RS:1;1cb9cf437a36:46291 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:1;1cb9cf437a36:46291 2024-11-28T06:56:48,439 INFO [RS:1;1cb9cf437a36:46291 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-28T06:56:48,439 INFO [RS:1;1cb9cf437a36:46291 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-28T06:56:48,439 DEBUG [RS:1;1cb9cf437a36:46291 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-28T06:56:48,441 INFO [RS:1;1cb9cf437a36:46291 {}] regionserver.HRegionServer(2659): reportForDuty to master=1cb9cf437a36,42945,1732777007591 with port=46291, startcode=1732777007811 2024-11-28T06:56:48,441 DEBUG [RS:1;1cb9cf437a36:46291 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-28T06:56:48,442 INFO [HMaster-EventLoopGroup-6-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34927, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=RegionServerStatusService 2024-11-28T06:56:48,443 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=42945 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 1cb9cf437a36,40661,1732777007755 2024-11-28T06:56:48,443 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=42945 {}] master.ServerManager(517): Registering regionserver=1cb9cf437a36,40661,1732777007755 2024-11-28T06:56:48,446 DEBUG [RS:0;1cb9cf437a36:40661 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7e9a7384-d065-9ff8-196a-59688d3a4920/752619e7-664e-6a55-3bd6-93cba9c9ba92 2024-11-28T06:56:48,446 DEBUG [RS:0;1cb9cf437a36:40661 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=file:/ 2024-11-28T06:56:48,446 DEBUG [RS:0;1cb9cf437a36:40661 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-28T06:56:48,451 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-28T06:56:48,451 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:56:48,452 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-28T06:56:48,452 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family rep_barrier of region 1588230740 2024-11-28T06:56:48,456 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-28T06:56:48,456 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:56:48,456 INFO [HMaster-EventLoopGroup-6-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49981, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=RegionServerStatusService 2024-11-28T06:56:48,456 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-28T06:56:48,457 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family table of region 1588230740 2024-11-28T06:56:48,457 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=42945 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 1cb9cf437a36,46291,1732777007811 2024-11-28T06:56:48,457 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=42945 {}] master.ServerManager(517): Registering regionserver=1cb9cf437a36,46291,1732777007811 2024-11-28T06:56:48,458 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42945-0x10181846c940000, quorum=127.0.0.1:61052, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-28T06:56:48,460 DEBUG [RS:1;1cb9cf437a36:46291 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7e9a7384-d065-9ff8-196a-59688d3a4920/752619e7-664e-6a55-3bd6-93cba9c9ba92 2024-11-28T06:56:48,460 DEBUG [RS:1;1cb9cf437a36:46291 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=file:/ 2024-11-28T06:56:48,460 DEBUG [RS:1;1cb9cf437a36:46291 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-28T06:56:48,460 DEBUG [RS:0;1cb9cf437a36:40661 {}] zookeeper.ZKUtil(111): regionserver:40661-0x10181846c940001, quorum=127.0.0.1:61052, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/1cb9cf437a36,40661,1732777007755 2024-11-28T06:56:48,461 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [1cb9cf437a36,40661,1732777007755] 2024-11-28T06:56:48,461 WARN [RS:0;1cb9cf437a36:40661 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-28T06:56:48,461 INFO [RS:0;1cb9cf437a36:40661 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-28T06:56:48,461 DEBUG [RS:0;1cb9cf437a36:40661 {}] regionserver.HRegionServer(1793): logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7e9a7384-d065-9ff8-196a-59688d3a4920/752619e7-664e-6a55-3bd6-93cba9c9ba92/WALs/1cb9cf437a36,40661,1732777007755 2024-11-28T06:56:48,463 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-28T06:56:48,463 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:56:48,464 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-28T06:56:48,464 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-28T06:56:48,464 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7e9a7384-d065-9ff8-196a-59688d3a4920/752619e7-664e-6a55-3bd6-93cba9c9ba92/data/hbase/meta/1588230740 2024-11-28T06:56:48,464 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7e9a7384-d065-9ff8-196a-59688d3a4920/752619e7-664e-6a55-3bd6-93cba9c9ba92/data/hbase/meta/1588230740 2024-11-28T06:56:48,465 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-28T06:56:48,465 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-28T06:56:48,465 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-28T06:56:48,466 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-28T06:56:48,475 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7e9a7384-d065-9ff8-196a-59688d3a4920/752619e7-664e-6a55-3bd6-93cba9c9ba92/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-28T06:56:48,475 INFO [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=71415553, jitterRate=0.06417466700077057}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-28T06:56:48,476 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740: Writing region info on filesystem at 1732777008402Initializing all the Stores at 1732777008403 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732777008403Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732777008406 (+3 ms)Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732777008406Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732777008406Cleaning up temporary data from old regions at 1732777008465 (+59 ms)Region opened successfully at 1732777008476 (+11 ms) 2024-11-28T06:56:48,477 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-28T06:56:48,477 INFO [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-28T06:56:48,477 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-28T06:56:48,477 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-28T06:56:48,477 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-28T06:56:48,477 INFO [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-28T06:56:48,478 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1732777008477Disabling compacts and flushes for region at 1732777008477Disabling writes for close at 1732777008477Writing region close event to WAL at 1732777008477Closed at 1732777008477 2024-11-28T06:56:48,479 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-28T06:56:48,479 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta 2024-11-28T06:56:48,479 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-11-28T06:56:48,480 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-28T06:56:48,481 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2024-11-28T06:56:48,490 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42945-0x10181846c940000, quorum=127.0.0.1:61052, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-28T06:56:48,491 DEBUG [RS:1;1cb9cf437a36:46291 {}] zookeeper.ZKUtil(111): regionserver:46291-0x10181846c940002, quorum=127.0.0.1:61052, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/1cb9cf437a36,46291,1732777007811 2024-11-28T06:56:48,491 WARN [RS:1;1cb9cf437a36:46291 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-28T06:56:48,491 INFO [RS:1;1cb9cf437a36:46291 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-28T06:56:48,492 DEBUG [RS:1;1cb9cf437a36:46291 {}] regionserver.HRegionServer(1793): logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7e9a7384-d065-9ff8-196a-59688d3a4920/752619e7-664e-6a55-3bd6-93cba9c9ba92/WALs/1cb9cf437a36,46291,1732777007811 2024-11-28T06:56:48,492 INFO [RS:0;1cb9cf437a36:40661 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-28T06:56:48,492 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [1cb9cf437a36,46291,1732777007811] 2024-11-28T06:56:48,498 INFO [RS:1;1cb9cf437a36:46291 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-28T06:56:48,499 INFO [RS:0;1cb9cf437a36:40661 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-28T06:56:48,499 INFO [RS:0;1cb9cf437a36:40661 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-28T06:56:48,499 INFO [RS:0;1cb9cf437a36:40661 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-28T06:56:48,500 INFO [RS:0;1cb9cf437a36:40661 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-28T06:56:48,501 INFO [RS:1;1cb9cf437a36:46291 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-28T06:56:48,501 INFO [RS:0;1cb9cf437a36:40661 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-28T06:56:48,502 INFO [RS:0;1cb9cf437a36:40661 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-28T06:56:48,502 DEBUG [RS:0;1cb9cf437a36:40661 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/1cb9cf437a36:0, corePoolSize=1, maxPoolSize=1 2024-11-28T06:56:48,502 DEBUG [RS:0;1cb9cf437a36:40661 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/1cb9cf437a36:0, corePoolSize=1, maxPoolSize=1 2024-11-28T06:56:48,502 DEBUG [RS:0;1cb9cf437a36:40661 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/1cb9cf437a36:0, corePoolSize=1, maxPoolSize=1 2024-11-28T06:56:48,502 DEBUG [RS:0;1cb9cf437a36:40661 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/1cb9cf437a36:0, corePoolSize=1, maxPoolSize=1 2024-11-28T06:56:48,502 DEBUG [RS:0;1cb9cf437a36:40661 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/1cb9cf437a36:0, corePoolSize=1, maxPoolSize=1 2024-11-28T06:56:48,502 DEBUG [RS:0;1cb9cf437a36:40661 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/1cb9cf437a36:0, corePoolSize=2, maxPoolSize=2 2024-11-28T06:56:48,502 DEBUG [RS:0;1cb9cf437a36:40661 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/1cb9cf437a36:0, corePoolSize=1, maxPoolSize=1 2024-11-28T06:56:48,503 DEBUG [RS:0;1cb9cf437a36:40661 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/1cb9cf437a36:0, corePoolSize=1, maxPoolSize=1 2024-11-28T06:56:48,503 DEBUG [RS:0;1cb9cf437a36:40661 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/1cb9cf437a36:0, corePoolSize=1, maxPoolSize=1 2024-11-28T06:56:48,503 DEBUG [RS:0;1cb9cf437a36:40661 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/1cb9cf437a36:0, corePoolSize=1, maxPoolSize=1 2024-11-28T06:56:48,503 DEBUG [RS:0;1cb9cf437a36:40661 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/1cb9cf437a36:0, corePoolSize=1, maxPoolSize=1 2024-11-28T06:56:48,503 DEBUG [RS:0;1cb9cf437a36:40661 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/1cb9cf437a36:0, corePoolSize=1, maxPoolSize=1 2024-11-28T06:56:48,503 DEBUG [RS:0;1cb9cf437a36:40661 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/1cb9cf437a36:0, corePoolSize=3, maxPoolSize=3 2024-11-28T06:56:48,503 DEBUG [RS:0;1cb9cf437a36:40661 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0, corePoolSize=3, maxPoolSize=3 2024-11-28T06:56:48,503 INFO [RS:1;1cb9cf437a36:46291 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-28T06:56:48,503 INFO [RS:1;1cb9cf437a36:46291 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-28T06:56:48,509 INFO [RS:1;1cb9cf437a36:46291 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-28T06:56:48,510 INFO [RS:1;1cb9cf437a36:46291 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-28T06:56:48,510 INFO [RS:0;1cb9cf437a36:40661 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-28T06:56:48,510 INFO [RS:0;1cb9cf437a36:40661 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-28T06:56:48,510 INFO [RS:1;1cb9cf437a36:46291 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-28T06:56:48,510 INFO [RS:0;1cb9cf437a36:40661 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-28T06:56:48,510 INFO [RS:0;1cb9cf437a36:40661 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-28T06:56:48,510 DEBUG [RS:1;1cb9cf437a36:46291 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/1cb9cf437a36:0, corePoolSize=1, maxPoolSize=1 2024-11-28T06:56:48,510 INFO [RS:0;1cb9cf437a36:40661 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-28T06:56:48,510 DEBUG [RS:1;1cb9cf437a36:46291 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/1cb9cf437a36:0, corePoolSize=1, maxPoolSize=1 2024-11-28T06:56:48,510 INFO [RS:0;1cb9cf437a36:40661 {}] hbase.ChoreService(168): Chore ScheduledChore name=1cb9cf437a36,40661,1732777007755-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-28T06:56:48,510 DEBUG [RS:1;1cb9cf437a36:46291 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/1cb9cf437a36:0, corePoolSize=1, maxPoolSize=1 2024-11-28T06:56:48,510 DEBUG [RS:1;1cb9cf437a36:46291 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/1cb9cf437a36:0, corePoolSize=1, maxPoolSize=1 2024-11-28T06:56:48,511 DEBUG [RS:1;1cb9cf437a36:46291 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/1cb9cf437a36:0, corePoolSize=1, maxPoolSize=1 2024-11-28T06:56:48,511 DEBUG [RS:1;1cb9cf437a36:46291 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/1cb9cf437a36:0, corePoolSize=2, maxPoolSize=2 2024-11-28T06:56:48,511 DEBUG [RS:1;1cb9cf437a36:46291 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/1cb9cf437a36:0, corePoolSize=1, maxPoolSize=1 2024-11-28T06:56:48,511 DEBUG [RS:1;1cb9cf437a36:46291 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/1cb9cf437a36:0, corePoolSize=1, maxPoolSize=1 2024-11-28T06:56:48,511 DEBUG [RS:1;1cb9cf437a36:46291 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/1cb9cf437a36:0, corePoolSize=1, maxPoolSize=1 2024-11-28T06:56:48,511 DEBUG [RS:1;1cb9cf437a36:46291 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/1cb9cf437a36:0, corePoolSize=1, maxPoolSize=1 2024-11-28T06:56:48,511 DEBUG [RS:1;1cb9cf437a36:46291 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/1cb9cf437a36:0, corePoolSize=1, maxPoolSize=1 2024-11-28T06:56:48,511 DEBUG [RS:1;1cb9cf437a36:46291 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/1cb9cf437a36:0, corePoolSize=1, maxPoolSize=1 2024-11-28T06:56:48,511 DEBUG [RS:1;1cb9cf437a36:46291 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/1cb9cf437a36:0, corePoolSize=3, maxPoolSize=3 2024-11-28T06:56:48,512 DEBUG [RS:1;1cb9cf437a36:46291 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0, corePoolSize=3, maxPoolSize=3 2024-11-28T06:56:48,519 INFO [RS:1;1cb9cf437a36:46291 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-28T06:56:48,519 INFO [RS:1;1cb9cf437a36:46291 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-28T06:56:48,519 INFO [RS:1;1cb9cf437a36:46291 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-28T06:56:48,519 INFO [RS:1;1cb9cf437a36:46291 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-28T06:56:48,520 INFO [RS:1;1cb9cf437a36:46291 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-28T06:56:48,520 INFO [RS:1;1cb9cf437a36:46291 {}] hbase.ChoreService(168): Chore ScheduledChore name=1cb9cf437a36,46291,1732777007811-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-28T06:56:48,534 INFO [RS:0;1cb9cf437a36:40661 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-28T06:56:48,534 INFO [RS:0;1cb9cf437a36:40661 {}] hbase.ChoreService(168): Chore ScheduledChore name=1cb9cf437a36,40661,1732777007755-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-28T06:56:48,535 INFO [RS:0;1cb9cf437a36:40661 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-28T06:56:48,535 INFO [RS:0;1cb9cf437a36:40661 {}] regionserver.Replication(171): 1cb9cf437a36,40661,1732777007755 started 2024-11-28T06:56:48,542 INFO [RS:1;1cb9cf437a36:46291 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-28T06:56:48,543 INFO [RS:1;1cb9cf437a36:46291 {}] hbase.ChoreService(168): Chore ScheduledChore name=1cb9cf437a36,46291,1732777007811-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-28T06:56:48,543 INFO [RS:1;1cb9cf437a36:46291 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-28T06:56:48,543 INFO [RS:1;1cb9cf437a36:46291 {}] regionserver.Replication(171): 1cb9cf437a36,46291,1732777007811 started 2024-11-28T06:56:48,555 INFO [RS:0;1cb9cf437a36:40661 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-28T06:56:48,555 INFO [RS:0;1cb9cf437a36:40661 {}] regionserver.HRegionServer(1482): Serving as 1cb9cf437a36,40661,1732777007755, RpcServer on 1cb9cf437a36/172.17.0.2:40661, sessionid=0x10181846c940001 2024-11-28T06:56:48,555 DEBUG [RS:0;1cb9cf437a36:40661 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-28T06:56:48,555 DEBUG [RS:0;1cb9cf437a36:40661 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 1cb9cf437a36,40661,1732777007755 2024-11-28T06:56:48,555 DEBUG [RS:0;1cb9cf437a36:40661 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '1cb9cf437a36,40661,1732777007755' 2024-11-28T06:56:48,555 DEBUG [RS:0;1cb9cf437a36:40661 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-28T06:56:48,556 DEBUG [RS:0;1cb9cf437a36:40661 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-28T06:56:48,557 DEBUG [RS:0;1cb9cf437a36:40661 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-28T06:56:48,557 DEBUG [RS:0;1cb9cf437a36:40661 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-28T06:56:48,557 DEBUG [RS:0;1cb9cf437a36:40661 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 1cb9cf437a36,40661,1732777007755 2024-11-28T06:56:48,557 DEBUG [RS:0;1cb9cf437a36:40661 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '1cb9cf437a36,40661,1732777007755' 2024-11-28T06:56:48,557 DEBUG [RS:0;1cb9cf437a36:40661 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-28T06:56:48,558 DEBUG [RS:0;1cb9cf437a36:40661 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-28T06:56:48,559 DEBUG [RS:0;1cb9cf437a36:40661 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-28T06:56:48,559 INFO [RS:0;1cb9cf437a36:40661 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-28T06:56:48,560 INFO [RS:0;1cb9cf437a36:40661 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-28T06:56:48,564 INFO [RS:1;1cb9cf437a36:46291 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-28T06:56:48,564 INFO [RS:1;1cb9cf437a36:46291 {}] regionserver.HRegionServer(1482): Serving as 1cb9cf437a36,46291,1732777007811, RpcServer on 1cb9cf437a36/172.17.0.2:46291, sessionid=0x10181846c940002 2024-11-28T06:56:48,564 DEBUG [RS:1;1cb9cf437a36:46291 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-28T06:56:48,564 DEBUG [RS:1;1cb9cf437a36:46291 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 1cb9cf437a36,46291,1732777007811 2024-11-28T06:56:48,564 DEBUG [RS:1;1cb9cf437a36:46291 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '1cb9cf437a36,46291,1732777007811' 2024-11-28T06:56:48,564 DEBUG [RS:1;1cb9cf437a36:46291 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-28T06:56:48,565 DEBUG [RS:1;1cb9cf437a36:46291 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-28T06:56:48,565 DEBUG [RS:1;1cb9cf437a36:46291 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-28T06:56:48,566 DEBUG [RS:1;1cb9cf437a36:46291 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-28T06:56:48,566 DEBUG [RS:1;1cb9cf437a36:46291 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 1cb9cf437a36,46291,1732777007811 2024-11-28T06:56:48,566 DEBUG [RS:1;1cb9cf437a36:46291 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '1cb9cf437a36,46291,1732777007811' 2024-11-28T06:56:48,566 DEBUG [RS:1;1cb9cf437a36:46291 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-28T06:56:48,566 DEBUG [RS:1;1cb9cf437a36:46291 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-28T06:56:48,567 DEBUG [RS:1;1cb9cf437a36:46291 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-28T06:56:48,567 INFO [RS:1;1cb9cf437a36:46291 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-28T06:56:48,567 INFO [RS:1;1cb9cf437a36:46291 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-28T06:56:48,631 WARN [1cb9cf437a36:42945 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-28T06:56:48,660 INFO [RS:0;1cb9cf437a36:40661 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-28T06:56:48,661 INFO [RS:0;1cb9cf437a36:40661 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=1cb9cf437a36%2C40661%2C1732777007755, suffix=, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7e9a7384-d065-9ff8-196a-59688d3a4920/752619e7-664e-6a55-3bd6-93cba9c9ba92/WALs/1cb9cf437a36,40661,1732777007755, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7e9a7384-d065-9ff8-196a-59688d3a4920/752619e7-664e-6a55-3bd6-93cba9c9ba92/oldWALs, maxLogs=55 2024-11-28T06:56:48,668 INFO [RS:1;1cb9cf437a36:46291 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-28T06:56:48,668 INFO [RS:1;1cb9cf437a36:46291 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=1cb9cf437a36%2C46291%2C1732777007811, suffix=, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7e9a7384-d065-9ff8-196a-59688d3a4920/752619e7-664e-6a55-3bd6-93cba9c9ba92/WALs/1cb9cf437a36,46291,1732777007811, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7e9a7384-d065-9ff8-196a-59688d3a4920/752619e7-664e-6a55-3bd6-93cba9c9ba92/oldWALs, maxLogs=55 2024-11-28T06:56:48,686 INFO [RS:0;1cb9cf437a36:40661 {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7e9a7384-d065-9ff8-196a-59688d3a4920/752619e7-664e-6a55-3bd6-93cba9c9ba92/WALs/1cb9cf437a36,40661,1732777007755/1cb9cf437a36%2C40661%2C1732777007755.1732777008663 2024-11-28T06:56:48,687 DEBUG [RS:0;1cb9cf437a36:40661 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-11-28T06:56:48,693 INFO [RS:1;1cb9cf437a36:46291 {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7e9a7384-d065-9ff8-196a-59688d3a4920/752619e7-664e-6a55-3bd6-93cba9c9ba92/WALs/1cb9cf437a36,46291,1732777007811/1cb9cf437a36%2C46291%2C1732777007811.1732777008670 2024-11-28T06:56:48,693 DEBUG [RS:1;1cb9cf437a36:46291 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-11-28T06:56:48,882 DEBUG [1cb9cf437a36:42945 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=2, allServersCount=2 2024-11-28T06:56:48,882 DEBUG [1cb9cf437a36:42945 {}] balancer.BalancerClusterState(204): Hosts are {1cb9cf437a36=0} racks are {/default-rack=0} 2024-11-28T06:56:48,883 DEBUG [1cb9cf437a36:42945 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-11-28T06:56:48,883 DEBUG [1cb9cf437a36:42945 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-11-28T06:56:48,883 DEBUG [1cb9cf437a36:42945 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-11-28T06:56:48,883 DEBUG [1cb9cf437a36:42945 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-11-28T06:56:48,883 INFO [1cb9cf437a36:42945 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-11-28T06:56:48,883 INFO [1cb9cf437a36:42945 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-11-28T06:56:48,883 DEBUG [1cb9cf437a36:42945 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-11-28T06:56:48,884 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=1cb9cf437a36,46291,1732777007811 2024-11-28T06:56:48,885 INFO [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 1cb9cf437a36,46291,1732777007811, state=OPENING 2024-11-28T06:56:48,908 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2024-11-28T06:56:48,917 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40661-0x10181846c940001, quorum=127.0.0.1:61052, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-28T06:56:48,918 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-28T06:56:48,927 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46291-0x10181846c940002, quorum=127.0.0.1:61052, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-28T06:56:48,927 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42945-0x10181846c940000, quorum=127.0.0.1:61052, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-28T06:56:48,927 DEBUG [PEWorker-3 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-28T06:56:48,927 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-28T06:56:48,927 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-28T06:56:48,927 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=1cb9cf437a36,46291,1732777007811}] 2024-11-28T06:56:49,081 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-28T06:56:49,084 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-8-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59029, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-28T06:56:49,086 INFO [RS_OPEN_META-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-11-28T06:56:49,086 INFO [RS_OPEN_META-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-28T06:56:49,087 INFO [RS_OPEN_META-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-11-28T06:56:49,088 INFO [RS_OPEN_META-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=1cb9cf437a36%2C46291%2C1732777007811.meta, suffix=.meta, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7e9a7384-d065-9ff8-196a-59688d3a4920/752619e7-664e-6a55-3bd6-93cba9c9ba92/WALs/1cb9cf437a36,46291,1732777007811, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7e9a7384-d065-9ff8-196a-59688d3a4920/752619e7-664e-6a55-3bd6-93cba9c9ba92/oldWALs, maxLogs=55 2024-11-28T06:56:49,112 INFO [RS_OPEN_META-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7e9a7384-d065-9ff8-196a-59688d3a4920/752619e7-664e-6a55-3bd6-93cba9c9ba92/WALs/1cb9cf437a36,46291,1732777007811/1cb9cf437a36%2C46291%2C1732777007811.meta.1732777009089.meta 2024-11-28T06:56:49,113 DEBUG [RS_OPEN_META-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-11-28T06:56:49,113 DEBUG [RS_OPEN_META-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-11-28T06:56:49,114 DEBUG [RS_OPEN_META-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-11-28T06:56:49,114 DEBUG [RS_OPEN_META-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-11-28T06:56:49,114 INFO [RS_OPEN_META-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-11-28T06:56:49,114 DEBUG [RS_OPEN_META-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-11-28T06:56:49,114 DEBUG [RS_OPEN_META-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-28T06:56:49,114 DEBUG [RS_OPEN_META-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-11-28T06:56:49,114 DEBUG [RS_OPEN_META-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-11-28T06:56:49,115 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family info of region 1588230740 2024-11-28T06:56:49,116 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-28T06:56:49,116 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:56:49,117 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-28T06:56:49,117 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family ns of region 1588230740 2024-11-28T06:56:49,118 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-28T06:56:49,118 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:56:49,118 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-28T06:56:49,118 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family rep_barrier of region 1588230740 2024-11-28T06:56:49,119 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-28T06:56:49,119 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:56:49,119 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-28T06:56:49,119 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family table of region 1588230740 2024-11-28T06:56:49,120 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-28T06:56:49,120 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:56:49,120 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-28T06:56:49,120 DEBUG [RS_OPEN_META-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-28T06:56:49,121 DEBUG [RS_OPEN_META-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7e9a7384-d065-9ff8-196a-59688d3a4920/752619e7-664e-6a55-3bd6-93cba9c9ba92/data/hbase/meta/1588230740 2024-11-28T06:56:49,122 DEBUG [RS_OPEN_META-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7e9a7384-d065-9ff8-196a-59688d3a4920/752619e7-664e-6a55-3bd6-93cba9c9ba92/data/hbase/meta/1588230740 2024-11-28T06:56:49,122 DEBUG [RS_OPEN_META-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-28T06:56:49,122 DEBUG [RS_OPEN_META-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-28T06:56:49,122 DEBUG [RS_OPEN_META-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-28T06:56:49,123 DEBUG [RS_OPEN_META-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-28T06:56:49,123 INFO [RS_OPEN_META-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=73326035, jitterRate=0.09264306724071503}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-28T06:56:49,124 DEBUG [RS_OPEN_META-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-11-28T06:56:49,124 DEBUG [RS_OPEN_META-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1732777009115Writing region info on filesystem at 1732777009115Initializing all the Stores at 1732777009115Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732777009115Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732777009115Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732777009115Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732777009115Cleaning up temporary data from old regions at 1732777009122 (+7 ms)Running coprocessor post-open hooks at 1732777009124 (+2 ms)Region opened successfully at 1732777009124 2024-11-28T06:56:49,125 INFO [RS_OPEN_META-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1732777009080 2024-11-28T06:56:49,128 DEBUG [RS_OPEN_META-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-11-28T06:56:49,128 INFO [RS_OPEN_META-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-11-28T06:56:49,128 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=1cb9cf437a36,46291,1732777007811 2024-11-28T06:56:49,129 INFO [PEWorker-4 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 1cb9cf437a36,46291,1732777007811, state=OPEN 2024-11-28T06:56:49,158 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46291-0x10181846c940002, quorum=127.0.0.1:61052, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-28T06:56:49,158 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40661-0x10181846c940001, quorum=127.0.0.1:61052, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-28T06:56:49,158 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42945-0x10181846c940000, quorum=127.0.0.1:61052, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-28T06:56:49,158 DEBUG [PEWorker-4 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=1cb9cf437a36,46291,1732777007811 2024-11-28T06:56:49,158 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-28T06:56:49,159 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-28T06:56:49,159 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-28T06:56:49,161 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2 2024-11-28T06:56:49,162 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=1cb9cf437a36,46291,1732777007811 in 231 msec 2024-11-28T06:56:49,164 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1 2024-11-28T06:56:49,164 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 683 msec 2024-11-28T06:56:49,165 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-28T06:56:49,165 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces 2024-11-28T06:56:49,169 DEBUG [PEWorker-1 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-28T06:56:49,169 DEBUG [PEWorker-1 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=1cb9cf437a36,46291,1732777007811, seqNum=-1] 2024-11-28T06:56:49,170 DEBUG [PEWorker-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-28T06:56:49,172 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-8-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47991, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-28T06:56:49,179 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 838 msec 2024-11-28T06:56:49,180 INFO [master/1cb9cf437a36:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1732777009179, completionTime=-1 2024-11-28T06:56:49,180 INFO [master/1cb9cf437a36:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=2; waited=0ms, expected min=2 server(s), max=2 server(s), master is running 2024-11-28T06:56:49,180 DEBUG [master/1cb9cf437a36:0:becomeActiveMaster {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-28T06:56:49,182 INFO [master/1cb9cf437a36:0:becomeActiveMaster {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-11-28T06:56:49,183 INFO [master/1cb9cf437a36:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732777069182 2024-11-28T06:56:49,183 INFO [master/1cb9cf437a36:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732777129183 2024-11-28T06:56:49,183 INFO [master/1cb9cf437a36:0:becomeActiveMaster {}] assignment.AssignmentManager(1783): Joined the cluster in 2 msec 2024-11-28T06:56:49,183 INFO [master/1cb9cf437a36:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=1cb9cf437a36,42945,1732777007591-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-28T06:56:49,183 INFO [master/1cb9cf437a36:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=1cb9cf437a36,42945,1732777007591-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-28T06:56:49,183 INFO [master/1cb9cf437a36:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=1cb9cf437a36,42945,1732777007591-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-28T06:56:49,183 INFO [master/1cb9cf437a36:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-1cb9cf437a36:42945, period=300000, unit=MILLISECONDS is enabled. 2024-11-28T06:56:49,183 INFO [master/1cb9cf437a36:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2024-11-28T06:56:49,184 INFO [master/1cb9cf437a36:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled. 2024-11-28T06:56:49,187 DEBUG [master/1cb9cf437a36:0.Chore.1 {}] janitor.CatalogJanitor(180): 2024-11-28T06:56:49,189 INFO [master/1cb9cf437a36:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 1.272sec 2024-11-28T06:56:49,190 INFO [master/1cb9cf437a36:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2024-11-28T06:56:49,190 INFO [master/1cb9cf437a36:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2024-11-28T06:56:49,190 INFO [master/1cb9cf437a36:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2024-11-28T06:56:49,190 INFO [master/1cb9cf437a36:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2024-11-28T06:56:49,190 INFO [master/1cb9cf437a36:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2024-11-28T06:56:49,190 INFO [master/1cb9cf437a36:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=1cb9cf437a36,42945,1732777007591-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-28T06:56:49,190 INFO [master/1cb9cf437a36:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=1cb9cf437a36,42945,1732777007591-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2024-11-28T06:56:49,191 DEBUG [master/1cb9cf437a36:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds 2024-11-28T06:56:49,191 INFO [master/1cb9cf437a36:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2024-11-28T06:56:49,191 INFO [master/1cb9cf437a36:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=1cb9cf437a36,42945,1732777007591-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-28T06:56:49,269 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1d9d15ba, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T06:56:49,269 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 1cb9cf437a36,42945,-1 for getting cluster id 2024-11-28T06:56:49,269 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-28T06:56:49,271 DEBUG [HMaster-EventLoopGroup-6-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f1e003c1-57de-44c5-9f50-31707ec95057' 2024-11-28T06:56:49,272 DEBUG [RPCClient-NioEventLoopGroup-5-6 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-28T06:56:49,272 DEBUG [RPCClient-NioEventLoopGroup-5-6 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f1e003c1-57de-44c5-9f50-31707ec95057" 2024-11-28T06:56:49,273 DEBUG [RPCClient-NioEventLoopGroup-5-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@f2d0deb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T06:56:49,273 DEBUG [RPCClient-NioEventLoopGroup-5-6 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [1cb9cf437a36,42945,-1] 2024-11-28T06:56:49,273 DEBUG [RPCClient-NioEventLoopGroup-5-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-28T06:56:49,274 DEBUG [RPCClient-NioEventLoopGroup-5-6 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T06:56:49,275 INFO [HMaster-EventLoopGroup-6-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36608, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-28T06:56:49,277 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@34126543, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T06:56:49,278 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-28T06:56:49,280 DEBUG [RPCClient-NioEventLoopGroup-5-7 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=1cb9cf437a36,46291,1732777007811, seqNum=-1] 2024-11-28T06:56:49,280 DEBUG [RPCClient-NioEventLoopGroup-5-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-28T06:56:49,283 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-8-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36748, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-28T06:56:49,287 INFO [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=1cb9cf437a36,42945,1732777007591 2024-11-28T06:56:49,287 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-28T06:56:49,289 DEBUG [RPCClient-NioEventLoopGroup-5-7 {}] client.AsyncConnectionImpl(321): The fetched master address is 1cb9cf437a36,42945,1732777007591 2024-11-28T06:56:49,290 DEBUG [RPCClient-NioEventLoopGroup-5-7 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@636a27a9 2024-11-28T06:56:49,290 DEBUG [RPCClient-NioEventLoopGroup-5-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-28T06:56:49,292 INFO [HMaster-EventLoopGroup-6-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36618, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-28T06:56:49,293 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42945 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'testBlockEvictionOnGracefulStop1732777009287', {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-28T06:56:49,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42945 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=testBlockEvictionOnGracefulStop1732777009287 2024-11-28T06:56:49,296 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=testBlockEvictionOnGracefulStop1732777009287 execute state=CREATE_TABLE_PRE_OPERATION 2024-11-28T06:56:49,296 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:56:49,296 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42945 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "testBlockEvictionOnGracefulStop1732777009287" procId is: 4 2024-11-28T06:56:49,297 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=testBlockEvictionOnGracefulStop1732777009287 execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-28T06:56:49,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42945 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-28T06:56:49,309 INFO [RegionOpenAndInit-testBlockEvictionOnGracefulStop1732777009287-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 31ba9c474ceabbeaab02529db668ff6e, NAME => 'testBlockEvictionOnGracefulStop1732777009287,,1732777009293.31ba9c474ceabbeaab02529db668ff6e.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testBlockEvictionOnGracefulStop1732777009287', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7e9a7384-d065-9ff8-196a-59688d3a4920/752619e7-664e-6a55-3bd6-93cba9c9ba92 2024-11-28T06:56:49,317 DEBUG [RegionOpenAndInit-testBlockEvictionOnGracefulStop1732777009287-pool-0 {}] regionserver.HRegion(898): Instantiated testBlockEvictionOnGracefulStop1732777009287,,1732777009293.31ba9c474ceabbeaab02529db668ff6e.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-28T06:56:49,317 DEBUG [RegionOpenAndInit-testBlockEvictionOnGracefulStop1732777009287-pool-0 {}] regionserver.HRegion(1722): Closing 31ba9c474ceabbeaab02529db668ff6e, disabling compactions & flushes 2024-11-28T06:56:49,317 INFO [RegionOpenAndInit-testBlockEvictionOnGracefulStop1732777009287-pool-0 {}] regionserver.HRegion(1755): Closing region testBlockEvictionOnGracefulStop1732777009287,,1732777009293.31ba9c474ceabbeaab02529db668ff6e. 2024-11-28T06:56:49,317 DEBUG [RegionOpenAndInit-testBlockEvictionOnGracefulStop1732777009287-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on testBlockEvictionOnGracefulStop1732777009287,,1732777009293.31ba9c474ceabbeaab02529db668ff6e. 2024-11-28T06:56:49,317 DEBUG [RegionOpenAndInit-testBlockEvictionOnGracefulStop1732777009287-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on testBlockEvictionOnGracefulStop1732777009287,,1732777009293.31ba9c474ceabbeaab02529db668ff6e. after waiting 0 ms 2024-11-28T06:56:49,317 DEBUG [RegionOpenAndInit-testBlockEvictionOnGracefulStop1732777009287-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region testBlockEvictionOnGracefulStop1732777009287,,1732777009293.31ba9c474ceabbeaab02529db668ff6e. 2024-11-28T06:56:49,317 INFO [RegionOpenAndInit-testBlockEvictionOnGracefulStop1732777009287-pool-0 {}] regionserver.HRegion(1973): Closed testBlockEvictionOnGracefulStop1732777009287,,1732777009293.31ba9c474ceabbeaab02529db668ff6e. 2024-11-28T06:56:49,317 DEBUG [RegionOpenAndInit-testBlockEvictionOnGracefulStop1732777009287-pool-0 {}] regionserver.HRegion(1676): Region close journal for 31ba9c474ceabbeaab02529db668ff6e: Waiting for close lock at 1732777009317Disabling compacts and flushes for region at 1732777009317Disabling writes for close at 1732777009317Writing region close event to WAL at 1732777009317Closed at 1732777009317 2024-11-28T06:56:49,318 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=testBlockEvictionOnGracefulStop1732777009287 execute state=CREATE_TABLE_ADD_TO_META 2024-11-28T06:56:49,319 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testBlockEvictionOnGracefulStop1732777009287,,1732777009293.31ba9c474ceabbeaab02529db668ff6e.","families":{"info":[{"qualifier":"regioninfo","vlen":78,"tag":[],"timestamp":"1732777009319"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732777009319"}]},"ts":"1732777009319"} 2024-11-28T06:56:49,322 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-28T06:56:49,323 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=testBlockEvictionOnGracefulStop1732777009287 execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-28T06:56:49,323 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testBlockEvictionOnGracefulStop1732777009287","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732777009323"}]},"ts":"1732777009323"} 2024-11-28T06:56:49,326 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=testBlockEvictionOnGracefulStop1732777009287, state=ENABLING in hbase:meta 2024-11-28T06:56:49,326 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(204): Hosts are {1cb9cf437a36=0} racks are {/default-rack=0} 2024-11-28T06:56:49,327 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-11-28T06:56:49,327 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-11-28T06:56:49,327 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-11-28T06:56:49,327 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-11-28T06:56:49,327 INFO [PEWorker-3 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-11-28T06:56:49,327 INFO [PEWorker-3 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-11-28T06:56:49,327 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-11-28T06:56:49,327 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1732777009287, region=31ba9c474ceabbeaab02529db668ff6e, ASSIGN}] 2024-11-28T06:56:49,329 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1732777009287, region=31ba9c474ceabbeaab02529db668ff6e, ASSIGN 2024-11-28T06:56:49,330 INFO [PEWorker-5 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1732777009287, region=31ba9c474ceabbeaab02529db668ff6e, ASSIGN; state=OFFLINE, location=1cb9cf437a36,46291,1732777007811; forceNewPlan=false, retain=false 2024-11-28T06:56:49,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42945 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-28T06:56:49,481 INFO [1cb9cf437a36:42945 {}] balancer.BaseLoadBalancer(388): Reassigned 1 regions. 1 retained the pre-restart assignment. 2024-11-28T06:56:49,482 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=31ba9c474ceabbeaab02529db668ff6e, regionState=OPENING, regionLocation=1cb9cf437a36,46291,1732777007811 2024-11-28T06:56:49,489 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-8-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1732777009287, region=31ba9c474ceabbeaab02529db668ff6e, ASSIGN because future has completed 2024-11-28T06:56:49,490 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 31ba9c474ceabbeaab02529db668ff6e, server=1cb9cf437a36,46291,1732777007811}] 2024-11-28T06:56:49,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42945 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-28T06:56:49,649 INFO [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(132): Open testBlockEvictionOnGracefulStop1732777009287,,1732777009293.31ba9c474ceabbeaab02529db668ff6e. 2024-11-28T06:56:49,649 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-28T06:56:49,650 DEBUG [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7752): Opening region: {ENCODED => 31ba9c474ceabbeaab02529db668ff6e, NAME => 'testBlockEvictionOnGracefulStop1732777009287,,1732777009293.31ba9c474ceabbeaab02529db668ff6e.', STARTKEY => '', ENDKEY => ''} 2024-11-28T06:56:49,651 DEBUG [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testBlockEvictionOnGracefulStop1732777009287 31ba9c474ceabbeaab02529db668ff6e 2024-11-28T06:56:49,651 DEBUG [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(898): Instantiated testBlockEvictionOnGracefulStop1732777009287,,1732777009293.31ba9c474ceabbeaab02529db668ff6e.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-28T06:56:49,651 DEBUG [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7794): checking encryption for 31ba9c474ceabbeaab02529db668ff6e 2024-11-28T06:56:49,651 DEBUG [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7797): checking classloading for 31ba9c474ceabbeaab02529db668ff6e 2024-11-28T06:56:49,652 INFO [StoreOpener-31ba9c474ceabbeaab02529db668ff6e-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family family of region 31ba9c474ceabbeaab02529db668ff6e 2024-11-28T06:56:49,656 INFO [StoreOpener-31ba9c474ceabbeaab02529db668ff6e-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 31ba9c474ceabbeaab02529db668ff6e columnFamilyName family 2024-11-28T06:56:49,656 DEBUG [StoreOpener-31ba9c474ceabbeaab02529db668ff6e-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:56:49,656 INFO [StoreOpener-31ba9c474ceabbeaab02529db668ff6e-1 {}] regionserver.HStore(327): Store=31ba9c474ceabbeaab02529db668ff6e/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-28T06:56:49,656 DEBUG [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1038): replaying wal for 31ba9c474ceabbeaab02529db668ff6e 2024-11-28T06:56:49,657 DEBUG [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7e9a7384-d065-9ff8-196a-59688d3a4920/752619e7-664e-6a55-3bd6-93cba9c9ba92/data/default/testBlockEvictionOnGracefulStop1732777009287/31ba9c474ceabbeaab02529db668ff6e 2024-11-28T06:56:49,657 DEBUG [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7e9a7384-d065-9ff8-196a-59688d3a4920/752619e7-664e-6a55-3bd6-93cba9c9ba92/data/default/testBlockEvictionOnGracefulStop1732777009287/31ba9c474ceabbeaab02529db668ff6e 2024-11-28T06:56:49,657 DEBUG [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1048): stopping wal replay for 31ba9c474ceabbeaab02529db668ff6e 2024-11-28T06:56:49,657 DEBUG [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1060): Cleaning up temporary data for 31ba9c474ceabbeaab02529db668ff6e 2024-11-28T06:56:49,658 DEBUG [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1093): writing seq id for 31ba9c474ceabbeaab02529db668ff6e 2024-11-28T06:56:49,669 DEBUG [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7e9a7384-d065-9ff8-196a-59688d3a4920/752619e7-664e-6a55-3bd6-93cba9c9ba92/data/default/testBlockEvictionOnGracefulStop1732777009287/31ba9c474ceabbeaab02529db668ff6e/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-28T06:56:49,669 INFO [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1114): Opened 31ba9c474ceabbeaab02529db668ff6e; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=58774191, jitterRate=-0.12419630587100983}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-28T06:56:49,669 DEBUG [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 31ba9c474ceabbeaab02529db668ff6e 2024-11-28T06:56:49,670 DEBUG [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1006): Region open journal for 31ba9c474ceabbeaab02529db668ff6e: Running coprocessor pre-open hook at 1732777009651Writing region info on filesystem at 1732777009651Initializing all the Stores at 1732777009651Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732777009651Cleaning up temporary data from old regions at 1732777009657 (+6 ms)Running coprocessor post-open hooks at 1732777009669 (+12 ms)Region opened successfully at 1732777009670 (+1 ms) 2024-11-28T06:56:49,671 INFO [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2236): Post open deploy tasks for testBlockEvictionOnGracefulStop1732777009287,,1732777009293.31ba9c474ceabbeaab02529db668ff6e., pid=6, masterSystemTime=1732777009643 2024-11-28T06:56:49,673 DEBUG [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2266): Finished post open deploy task for testBlockEvictionOnGracefulStop1732777009287,,1732777009293.31ba9c474ceabbeaab02529db668ff6e. 2024-11-28T06:56:49,673 INFO [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(153): Opened testBlockEvictionOnGracefulStop1732777009287,,1732777009293.31ba9c474ceabbeaab02529db668ff6e. 2024-11-28T06:56:49,674 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=31ba9c474ceabbeaab02529db668ff6e, regionState=OPEN, openSeqNum=2, regionLocation=1cb9cf437a36,46291,1732777007811 2024-11-28T06:56:49,676 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-8-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 31ba9c474ceabbeaab02529db668ff6e, server=1cb9cf437a36,46291,1732777007811 because future has completed 2024-11-28T06:56:49,679 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=6, resume processing ppid=5 2024-11-28T06:56:49,679 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure 31ba9c474ceabbeaab02529db668ff6e, server=1cb9cf437a36,46291,1732777007811 in 188 msec 2024-11-28T06:56:49,681 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=5, resume processing ppid=4 2024-11-28T06:56:49,681 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1732777009287, region=31ba9c474ceabbeaab02529db668ff6e, ASSIGN in 352 msec 2024-11-28T06:56:49,682 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=testBlockEvictionOnGracefulStop1732777009287 execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-28T06:56:49,682 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testBlockEvictionOnGracefulStop1732777009287","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732777009682"}]},"ts":"1732777009682"} 2024-11-28T06:56:49,685 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=testBlockEvictionOnGracefulStop1732777009287, state=ENABLED in hbase:meta 2024-11-28T06:56:49,686 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=testBlockEvictionOnGracefulStop1732777009287 execute state=CREATE_TABLE_POST_OPERATION 2024-11-28T06:56:49,687 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=testBlockEvictionOnGracefulStop1732777009287 in 392 msec 2024-11-28T06:56:49,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42945 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-28T06:56:49,925 INFO [RPCClient-NioEventLoopGroup-5-9 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:testBlockEvictionOnGracefulStop1732777009287 completed 2024-11-28T06:56:49,925 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table testBlockEvictionOnGracefulStop1732777009287 get assigned. Timeout = 60000ms 2024-11-28T06:56:49,926 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-28T06:56:49,936 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table testBlockEvictionOnGracefulStop1732777009287 assigned to meta. Checking AM states. 2024-11-28T06:56:49,937 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-28T06:56:49,937 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table testBlockEvictionOnGracefulStop1732777009287 assigned. 2024-11-28T06:56:49,940 DEBUG [RPCClient-NioEventLoopGroup-5-8 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testBlockEvictionOnGracefulStop1732777009287', row='row1', locateType=CURRENT is [region=testBlockEvictionOnGracefulStop1732777009287,,1732777009293.31ba9c474ceabbeaab02529db668ff6e., hostname=1cb9cf437a36,46291,1732777007811, seqNum=2] 2024-11-28T06:56:49,945 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 31ba9c474ceabbeaab02529db668ff6e 1/1 column families, dataSize=78 B heapSize=480 B 2024-11-28T06:56:49,953 DEBUG [Time-limited test {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 68fcf27268094562965c0951b6ee0334 2024-11-28T06:56:49,967 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7e9a7384-d065-9ff8-196a-59688d3a4920/752619e7-664e-6a55-3bd6-93cba9c9ba92/data/default/testBlockEvictionOnGracefulStop1732777009287/31ba9c474ceabbeaab02529db668ff6e/.tmp/family/68fcf27268094562965c0951b6ee0334 is 43, key is row1/family:qf1/1/Put/seqid=0 2024-11-28T06:56:49,967 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=78 B at sequenceid=6 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7e9a7384-d065-9ff8-196a-59688d3a4920/752619e7-664e-6a55-3bd6-93cba9c9ba92/data/default/testBlockEvictionOnGracefulStop1732777009287/31ba9c474ceabbeaab02529db668ff6e/.tmp/family/68fcf27268094562965c0951b6ee0334 2024-11-28T06:56:49,970 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7e9a7384-d065-9ff8-196a-59688d3a4920/752619e7-664e-6a55-3bd6-93cba9c9ba92/data/default/testBlockEvictionOnGracefulStop1732777009287/31ba9c474ceabbeaab02529db668ff6e/.tmp/family/68fcf27268094562965c0951b6ee0334 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7e9a7384-d065-9ff8-196a-59688d3a4920/752619e7-664e-6a55-3bd6-93cba9c9ba92/data/default/testBlockEvictionOnGracefulStop1732777009287/31ba9c474ceabbeaab02529db668ff6e/family/68fcf27268094562965c0951b6ee0334 2024-11-28T06:56:49,971 DEBUG [Time-limited test {}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7e9a7384-d065-9ff8-196a-59688d3a4920/752619e7-664e-6a55-3bd6-93cba9c9ba92/data/default/testBlockEvictionOnGracefulStop1732777009287/31ba9c474ceabbeaab02529db668ff6e/family/68fcf27268094562965c0951b6ee0334, delay=903 ms 2024-11-28T06:56:49,972 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7e9a7384-d065-9ff8-196a-59688d3a4920/752619e7-664e-6a55-3bd6-93cba9c9ba92/data/default/testBlockEvictionOnGracefulStop1732777009287/31ba9c474ceabbeaab02529db668ff6e/family/68fcf27268094562965c0951b6ee0334, entries=2, sequenceid=6, filesize=4.9 K 2024-11-28T06:56:49,973 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~78 B/78, heapSize ~464 B/464, currentSize=0 B/0 for 31ba9c474ceabbeaab02529db668ff6e in 27ms, sequenceid=6, compaction requested=false 2024-11-28T06:56:49,973 DEBUG [Time-limited test {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'testBlockEvictionOnGracefulStop1732777009287' 2024-11-28T06:56:49,973 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for 31ba9c474ceabbeaab02529db668ff6e: 2024-11-28T06:56:50,598 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testBlockEvictionOnGracefulStop1732777009287 2024-11-28T06:56:50,599 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testBlockEvictionOnGracefulStop1732777009287 Metrics about Tables on a single HBase RegionServer 2024-11-28T06:56:50,603 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta 2024-11-28T06:56:50,603 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_meta Metrics about Tables on a single HBase RegionServer 2024-11-28T06:56:50,605 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-28T06:56:50,605 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint Metrics about HBase RegionObservers 2024-11-28T06:56:50,606 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_master_table_store 2024-11-28T06:56:50,606 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_master_table_store Metrics about Tables on a single HBase RegionServer 2024-11-28T06:56:50,875 DEBUG [hfile-prefetch-1732777006907 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 68fcf27268094562965c0951b6ee0334_0. 2024-11-28T06:56:50,876 DEBUG [hfile-prefetch-1732777006907 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 68fcf27268094562965c0951b6ee0334_117. 2024-11-28T06:56:50,876 DEBUG [hfile-prefetch-1732777006907 {}] bucket.BucketCache(2281): Notifying caching completed for file file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7e9a7384-d065-9ff8-196a-59688d3a4920/752619e7-664e-6a55-3bd6-93cba9c9ba92/data/default/testBlockEvictionOnGracefulStop1732777009287/31ba9c474ceabbeaab02529db668ff6e/family/68fcf27268094562965c0951b6ee0334, with total blocks 2, and data blocks 1 2024-11-28T06:56:50,877 DEBUG [hfile-prefetch-1732777006907 {}] bucket.BucketCache(2285): iterating over 0 entries in the backing map 2024-11-28T06:56:50,877 DEBUG [hfile-prefetch-1732777006907 {}] bucket.BucketCache(2308): Prefetch executor completed for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7e9a7384-d065-9ff8-196a-59688d3a4920/752619e7-664e-6a55-3bd6-93cba9c9ba92/data/default/testBlockEvictionOnGracefulStop1732777009287/31ba9c474ceabbeaab02529db668ff6e/family/68fcf27268094562965c0951b6ee0334, but only 0 data blocks were cached. Total data blocks for file: 1. Checking for blocks pending cache in cache writer queue. 2024-11-28T06:56:50,877 DEBUG [hfile-prefetch-1732777006907 {}] bucket.BucketCache(2318): There are still blocks pending caching for file file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7e9a7384-d065-9ff8-196a-59688d3a4920/752619e7-664e-6a55-3bd6-93cba9c9ba92/data/default/testBlockEvictionOnGracefulStop1732777009287/31ba9c474ceabbeaab02529db668ff6e/family/68fcf27268094562965c0951b6ee0334. Will sleep 100ms and try the verification again. 2024-11-28T06:56:50,957 DEBUG [bucket-cache-persister {}] bucket.BucketCachePersister(47): Cache is inconsistent, persisting to disk 2024-11-28T06:56:50,957 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1487): Thread bucket-cache-persister started persisting bucket cache to file 2024-11-28T06:56:50,957 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1494): Persist in new chunked persistence format. 2024-11-28T06:56:50,957 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1673): persistToFile: before persisting backing map size: 1, fullycachedFiles size: 0, chunkSize: 10000 2024-11-28T06:56:50,972 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1680): persistToFile: after persisting backing map size: 1, fullycachedFiles size: 0 2024-11-28T06:56:50,972 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1498): PersistToFile: after persisting backing map size: 1, fullycachedFiles size: 0, file name: bucket.persistence1732777010957 2024-11-28T06:56:50,972 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1509): Thread bucket-cache-persister finished persisting bucket cache to file, renaming 2024-11-28T06:56:50,974 INFO [Time-limited test {}] hbase.SingleProcessHBaseCluster(504): Stopping Thread[RS:1;1cb9cf437a36:46291,5,FailOnTimeoutGroup] 2024-11-28T06:56:50,974 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '1cb9cf437a36,46291,1732777007811' ***** 2024-11-28T06:56:50,974 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Stopping rs 1 2024-11-28T06:56:50,974 INFO [RS:1;1cb9cf437a36:46291 {}] bucket.BucketCache(1738): Disabling cache 2024-11-28T06:56:50,974 INFO [RS:1;1cb9cf437a36:46291 {}] bucket.BucketCache(1762): Shutdown bucket cache: IO persistent=true; path to write=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/33d1270a-7ecb-879e-376a-4fd885e9a050/bucket.persistence 2024-11-28T06:56:50,974 INFO [Time-limited test-BucketCacheWriter-2 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-2 exiting, cacheEnabled=false 2024-11-28T06:56:50,974 INFO [Time-limited test-BucketCacheWriter-1 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-1 exiting, cacheEnabled=false 2024-11-28T06:56:50,974 INFO [Time-limited test-BucketCacheWriter-0 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-0 exiting, cacheEnabled=false 2024-11-28T06:56:50,974 INFO [RS:1;1cb9cf437a36:46291 {}] bucket.BucketCache(1768): Shutting down cache persister thread. 2024-11-28T06:56:50,975 WARN [bucket-cache-persister {}] bucket.BucketCachePersister(64): Interrupting BucketCachePersister thread. java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCachePersister.run(BucketCachePersister.java:45) ~[classes/:?] 2024-11-28T06:56:50,977 DEBUG [hfile-prefetch-1732777006907 {}] bucket.BucketCache(2281): Notifying caching completed for file file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7e9a7384-d065-9ff8-196a-59688d3a4920/752619e7-664e-6a55-3bd6-93cba9c9ba92/data/default/testBlockEvictionOnGracefulStop1732777009287/31ba9c474ceabbeaab02529db668ff6e/family/68fcf27268094562965c0951b6ee0334, with total blocks 2, and data blocks 1 2024-11-28T06:56:50,977 DEBUG [hfile-prefetch-1732777006907 {}] bucket.BucketCache(2285): iterating over 1 entries in the backing map 2024-11-28T06:56:50,977 DEBUG [hfile-prefetch-1732777006907 {}] bucket.BucketCache(2293): found block for file 68fcf27268094562965c0951b6ee0334 in the backing map. Acquiring read lock for offset 0 2024-11-28T06:56:50,977 DEBUG [hfile-prefetch-1732777006907 {}] bucket.BucketCache(2305): File file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7e9a7384-d065-9ff8-196a-59688d3a4920/752619e7-664e-6a55-3bd6-93cba9c9ba92/data/default/testBlockEvictionOnGracefulStop1732777009287/31ba9c474ceabbeaab02529db668ff6e/family/68fcf27268094562965c0951b6ee0334 has now been fully cached. 2024-11-28T06:56:50,978 DEBUG [hfile-prefetch-1732777006907 {}] hfile.PrefetchExecutor(121): Prefetch completed for 68fcf27268094562965c0951b6ee0334 2024-11-28T06:56:50,985 DEBUG [RS:1;1cb9cf437a36:46291 {}] bucket.BucketCache(1487): Thread RS:1;1cb9cf437a36:46291 started persisting bucket cache to file 2024-11-28T06:56:50,985 DEBUG [RS:1;1cb9cf437a36:46291 {}] bucket.BucketCache(1494): Persist in new chunked persistence format. 2024-11-28T06:56:50,985 DEBUG [RS:1;1cb9cf437a36:46291 {}] bucket.BucketCache(1673): persistToFile: before persisting backing map size: 1, fullycachedFiles size: 1, chunkSize: 10000 2024-11-28T06:56:50,988 DEBUG [RS:1;1cb9cf437a36:46291 {}] bucket.BucketCache(1680): persistToFile: after persisting backing map size: 1, fullycachedFiles size: 1 2024-11-28T06:56:50,988 DEBUG [RS:1;1cb9cf437a36:46291 {}] bucket.BucketCache(1498): PersistToFile: after persisting backing map size: 1, fullycachedFiles size: 1, file name: bucket.persistence1732777010985 2024-11-28T06:56:50,988 DEBUG [RS:1;1cb9cf437a36:46291 {}] bucket.BucketCache(1509): Thread RS:1;1cb9cf437a36:46291 finished persisting bucket cache to file, renaming 2024-11-28T06:56:50,989 INFO [RS:1;1cb9cf437a36:46291 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-28T06:56:50,989 INFO [RS:1;1cb9cf437a36:46291 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-28T06:56:50,989 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-28T06:56:50,989 INFO [RS:1;1cb9cf437a36:46291 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-28T06:56:50,989 INFO [RS:1;1cb9cf437a36:46291 {}] regionserver.HRegionServer(3091): Received CLOSE for 31ba9c474ceabbeaab02529db668ff6e 2024-11-28T06:56:50,989 INFO [RS:1;1cb9cf437a36:46291 {}] regionserver.HRegionServer(959): stopping server 1cb9cf437a36,46291,1732777007811 2024-11-28T06:56:50,989 INFO [RS:1;1cb9cf437a36:46291 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-28T06:56:50,989 INFO [RS:1;1cb9cf437a36:46291 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:1;1cb9cf437a36:46291. 2024-11-28T06:56:50,989 INFO [RS_CLOSE_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1662): Closing region testBlockEvictionOnGracefulStop1732777009287,,1732777009293.31ba9c474ceabbeaab02529db668ff6e. during a graceful stop, and cache persistence is on, so setting evict on close to false. 2024-11-28T06:56:50,989 DEBUG [RS:1;1cb9cf437a36:46291 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-28T06:56:50,989 DEBUG [RS_CLOSE_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 31ba9c474ceabbeaab02529db668ff6e, disabling compactions & flushes 2024-11-28T06:56:50,989 DEBUG [RS:1;1cb9cf437a36:46291 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T06:56:50,989 INFO [RS_CLOSE_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testBlockEvictionOnGracefulStop1732777009287,,1732777009293.31ba9c474ceabbeaab02529db668ff6e. 2024-11-28T06:56:50,989 DEBUG [RS_CLOSE_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testBlockEvictionOnGracefulStop1732777009287,,1732777009293.31ba9c474ceabbeaab02529db668ff6e. 2024-11-28T06:56:50,989 INFO [RS:1;1cb9cf437a36:46291 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-28T06:56:50,989 DEBUG [RS_CLOSE_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testBlockEvictionOnGracefulStop1732777009287,,1732777009293.31ba9c474ceabbeaab02529db668ff6e. after waiting 0 ms 2024-11-28T06:56:50,990 INFO [RS:1;1cb9cf437a36:46291 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-28T06:56:50,990 DEBUG [RS_CLOSE_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testBlockEvictionOnGracefulStop1732777009287,,1732777009293.31ba9c474ceabbeaab02529db668ff6e. 2024-11-28T06:56:50,990 INFO [RS:1;1cb9cf437a36:46291 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-28T06:56:50,990 INFO [RS:1;1cb9cf437a36:46291 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-11-28T06:56:50,990 INFO [RS:1;1cb9cf437a36:46291 {}] regionserver.HRegionServer(1321): Waiting on 2 regions to close 2024-11-28T06:56:50,990 DEBUG [RS:1;1cb9cf437a36:46291 {}] regionserver.HRegionServer(1325): Online Regions={1588230740=hbase:meta,,1.1588230740, 31ba9c474ceabbeaab02529db668ff6e=testBlockEvictionOnGracefulStop1732777009287,,1732777009293.31ba9c474ceabbeaab02529db668ff6e.} 2024-11-28T06:56:50,990 DEBUG [RS:1;1cb9cf437a36:46291 {}] regionserver.HRegionServer(1351): Waiting on 1588230740, 31ba9c474ceabbeaab02529db668ff6e 2024-11-28T06:56:50,990 INFO [RS_CLOSE_META-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1662): Closing region hbase:meta,,1.1588230740 during a graceful stop, and cache persistence is on, so setting evict on close to false. 2024-11-28T06:56:50,990 DEBUG [RS_CLOSE_META-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-28T06:56:50,990 INFO [RS_CLOSE_META-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-28T06:56:50,990 DEBUG [RS_CLOSE_META-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-28T06:56:50,990 DEBUG [RS_CLOSE_META-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-28T06:56:50,990 DEBUG [RS_CLOSE_META-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-28T06:56:50,990 INFO [RS_CLOSE_META-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=1.75 KB heapSize=3.78 KB 2024-11-28T06:56:50,995 DEBUG [RS_CLOSE_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7e9a7384-d065-9ff8-196a-59688d3a4920/752619e7-664e-6a55-3bd6-93cba9c9ba92/data/default/testBlockEvictionOnGracefulStop1732777009287/31ba9c474ceabbeaab02529db668ff6e/recovered.edits/9.seqid, newMaxSeqId=9, maxSeqId=1 2024-11-28T06:56:50,996 INFO [RS_CLOSE_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testBlockEvictionOnGracefulStop1732777009287,,1732777009293.31ba9c474ceabbeaab02529db668ff6e. 2024-11-28T06:56:50,996 DEBUG [RS_CLOSE_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 31ba9c474ceabbeaab02529db668ff6e: Waiting for close lock at 1732777010989Running coprocessor pre-close hooks at 1732777010989Disabling compacts and flushes for region at 1732777010989Disabling writes for close at 1732777010990 (+1 ms)Writing region close event to WAL at 1732777010991 (+1 ms)Running coprocessor post-close hooks at 1732777010996 (+5 ms)Closed at 1732777010996 2024-11-28T06:56:50,996 DEBUG [RS_CLOSE_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testBlockEvictionOnGracefulStop1732777009287,,1732777009293.31ba9c474ceabbeaab02529db668ff6e. 2024-11-28T06:56:50,999 DEBUG [RS_CLOSE_META-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 666397d4cded4a028775335b64d1e84e 2024-11-28T06:56:51,012 DEBUG [RS_CLOSE_META-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7e9a7384-d065-9ff8-196a-59688d3a4920/752619e7-664e-6a55-3bd6-93cba9c9ba92/data/hbase/meta/1588230740/.tmp/info/666397d4cded4a028775335b64d1e84e is 209, key is testBlockEvictionOnGracefulStop1732777009287,,1732777009293.31ba9c474ceabbeaab02529db668ff6e./info:regioninfo/1732777009673/Put/seqid=0 2024-11-28T06:56:51,012 INFO [RS_CLOSE_META-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.53 KB at sequenceid=11 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7e9a7384-d065-9ff8-196a-59688d3a4920/752619e7-664e-6a55-3bd6-93cba9c9ba92/data/hbase/meta/1588230740/.tmp/info/666397d4cded4a028775335b64d1e84e 2024-11-28T06:56:51,020 DEBUG [RS_CLOSE_META-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 4ef821c80f9a4e05ab163ae8a76012e3 2024-11-28T06:56:51,025 INFO [regionserver/1cb9cf437a36:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-28T06:56:51,031 DEBUG [RS_CLOSE_META-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7e9a7384-d065-9ff8-196a-59688d3a4920/752619e7-664e-6a55-3bd6-93cba9c9ba92/data/hbase/meta/1588230740/.tmp/ns/4ef821c80f9a4e05ab163ae8a76012e3 is 43, key is default/ns:d/1732777009172/Put/seqid=0 2024-11-28T06:56:51,032 INFO [RS_CLOSE_META-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=74 B at sequenceid=11 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7e9a7384-d065-9ff8-196a-59688d3a4920/752619e7-664e-6a55-3bd6-93cba9c9ba92/data/hbase/meta/1588230740/.tmp/ns/4ef821c80f9a4e05ab163ae8a76012e3 2024-11-28T06:56:51,040 DEBUG [RS_CLOSE_META-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 9b2097e829dc4962a8584e9c1d2359d8 2024-11-28T06:56:51,052 DEBUG [RS_CLOSE_META-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7e9a7384-d065-9ff8-196a-59688d3a4920/752619e7-664e-6a55-3bd6-93cba9c9ba92/data/hbase/meta/1588230740/.tmp/table/9b2097e829dc4962a8584e9c1d2359d8 is 80, key is testBlockEvictionOnGracefulStop1732777009287/table:state/1732777009682/Put/seqid=0 2024-11-28T06:56:51,053 INFO [RS_CLOSE_META-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=152 B at sequenceid=11 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7e9a7384-d065-9ff8-196a-59688d3a4920/752619e7-664e-6a55-3bd6-93cba9c9ba92/data/hbase/meta/1588230740/.tmp/table/9b2097e829dc4962a8584e9c1d2359d8 2024-11-28T06:56:51,055 DEBUG [RS_CLOSE_META-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7e9a7384-d065-9ff8-196a-59688d3a4920/752619e7-664e-6a55-3bd6-93cba9c9ba92/data/hbase/meta/1588230740/.tmp/info/666397d4cded4a028775335b64d1e84e as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7e9a7384-d065-9ff8-196a-59688d3a4920/752619e7-664e-6a55-3bd6-93cba9c9ba92/data/hbase/meta/1588230740/info/666397d4cded4a028775335b64d1e84e 2024-11-28T06:56:51,057 DEBUG [RS_CLOSE_META-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_CLOSE_META}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7e9a7384-d065-9ff8-196a-59688d3a4920/752619e7-664e-6a55-3bd6-93cba9c9ba92/data/hbase/meta/1588230740/info/666397d4cded4a028775335b64d1e84e, delay=969 ms 2024-11-28T06:56:51,058 INFO [RS_CLOSE_META-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7e9a7384-d065-9ff8-196a-59688d3a4920/752619e7-664e-6a55-3bd6-93cba9c9ba92/data/hbase/meta/1588230740/info/666397d4cded4a028775335b64d1e84e, entries=10, sequenceid=11, filesize=7.0 K 2024-11-28T06:56:51,058 DEBUG [RS_CLOSE_META-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7e9a7384-d065-9ff8-196a-59688d3a4920/752619e7-664e-6a55-3bd6-93cba9c9ba92/data/hbase/meta/1588230740/.tmp/ns/4ef821c80f9a4e05ab163ae8a76012e3 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7e9a7384-d065-9ff8-196a-59688d3a4920/752619e7-664e-6a55-3bd6-93cba9c9ba92/data/hbase/meta/1588230740/ns/4ef821c80f9a4e05ab163ae8a76012e3 2024-11-28T06:56:51,059 DEBUG [RS_CLOSE_META-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_CLOSE_META}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7e9a7384-d065-9ff8-196a-59688d3a4920/752619e7-664e-6a55-3bd6-93cba9c9ba92/data/hbase/meta/1588230740/ns/4ef821c80f9a4e05ab163ae8a76012e3, delay=927 ms 2024-11-28T06:56:51,060 INFO [RS_CLOSE_META-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7e9a7384-d065-9ff8-196a-59688d3a4920/752619e7-664e-6a55-3bd6-93cba9c9ba92/data/hbase/meta/1588230740/ns/4ef821c80f9a4e05ab163ae8a76012e3, entries=2, sequenceid=11, filesize=5.0 K 2024-11-28T06:56:51,060 DEBUG [RS_CLOSE_META-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7e9a7384-d065-9ff8-196a-59688d3a4920/752619e7-664e-6a55-3bd6-93cba9c9ba92/data/hbase/meta/1588230740/.tmp/table/9b2097e829dc4962a8584e9c1d2359d8 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7e9a7384-d065-9ff8-196a-59688d3a4920/752619e7-664e-6a55-3bd6-93cba9c9ba92/data/hbase/meta/1588230740/table/9b2097e829dc4962a8584e9c1d2359d8 2024-11-28T06:56:51,062 DEBUG [RS_CLOSE_META-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_CLOSE_META}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7e9a7384-d065-9ff8-196a-59688d3a4920/752619e7-664e-6a55-3bd6-93cba9c9ba92/data/hbase/meta/1588230740/table/9b2097e829dc4962a8584e9c1d2359d8, delay=984 ms 2024-11-28T06:56:51,063 INFO [RS_CLOSE_META-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7e9a7384-d065-9ff8-196a-59688d3a4920/752619e7-664e-6a55-3bd6-93cba9c9ba92/data/hbase/meta/1588230740/table/9b2097e829dc4962a8584e9c1d2359d8, entries=2, sequenceid=11, filesize=5.3 K 2024-11-28T06:56:51,063 INFO [RS_CLOSE_META-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~1.75 KB/1797, heapSize ~3.48 KB/3568, currentSize=0 B/0 for 1588230740 in 73ms, sequenceid=11, compaction requested=false 2024-11-28T06:56:51,064 DEBUG [StoreFileCloser-1588230740-info-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7e9a7384-d065-9ff8-196a-59688d3a4920/752619e7-664e-6a55-3bd6-93cba9c9ba92/data/hbase/meta/1588230740/info/666397d4cded4a028775335b64d1e84e 2024-11-28T06:56:51,065 DEBUG [StoreFileCloser-1588230740-ns-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7e9a7384-d065-9ff8-196a-59688d3a4920/752619e7-664e-6a55-3bd6-93cba9c9ba92/data/hbase/meta/1588230740/ns/4ef821c80f9a4e05ab163ae8a76012e3 2024-11-28T06:56:51,065 DEBUG [StoreFileCloser-1588230740-table-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7e9a7384-d065-9ff8-196a-59688d3a4920/752619e7-664e-6a55-3bd6-93cba9c9ba92/data/hbase/meta/1588230740/table/9b2097e829dc4962a8584e9c1d2359d8 2024-11-28T06:56:51,069 DEBUG [RS_CLOSE_META-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7e9a7384-d065-9ff8-196a-59688d3a4920/752619e7-664e-6a55-3bd6-93cba9c9ba92/data/hbase/meta/1588230740/recovered.edits/14.seqid, newMaxSeqId=14, maxSeqId=1 2024-11-28T06:56:51,069 DEBUG [RS_CLOSE_META-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-28T06:56:51,069 INFO [RS_CLOSE_META-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-28T06:56:51,070 DEBUG [RS_CLOSE_META-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1732777010990Running coprocessor pre-close hooks at 1732777010990Disabling compacts and flushes for region at 1732777010990Disabling writes for close at 1732777010990Obtaining lock to block concurrent updates at 1732777010990Preparing flush snapshotting stores in 1588230740 at 1732777010990Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=1797, getHeapSize=3808, getOffHeapSize=0, getCellsCount=14 at 1732777010991 (+1 ms)Flushing stores of hbase:meta,,1.1588230740 at 1732777010991Flushing 1588230740/info: creating writer at 1732777010991Flushing 1588230740/info: appending metadata at 1732777011011 (+20 ms)Flushing 1588230740/info: closing flushed file at 1732777011011Flushing 1588230740/ns: creating writer at 1732777011014 (+3 ms)Flushing 1588230740/ns: appending metadata at 1732777011031 (+17 ms)Flushing 1588230740/ns: closing flushed file at 1732777011031Flushing 1588230740/table: creating writer at 1732777011034 (+3 ms)Flushing 1588230740/table: appending metadata at 1732777011052 (+18 ms)Flushing 1588230740/table: closing flushed file at 1732777011052Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@78ced686: reopening flushed file at 1732777011055 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@6365179d: reopening flushed file at 1732777011058 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@4630b288: reopening flushed file at 1732777011060 (+2 ms)Finished flush of dataSize ~1.75 KB/1797, heapSize ~3.48 KB/3568, currentSize=0 B/0 for 1588230740 in 73ms, sequenceid=11, compaction requested=false at 1732777011064 (+4 ms)Writing region close event to WAL at 1732777011066 (+2 ms)Running coprocessor post-close hooks at 1732777011069 (+3 ms)Closed at 1732777011069 2024-11-28T06:56:51,070 DEBUG [RS_CLOSE_META-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-11-28T06:56:51,190 INFO [RS:1;1cb9cf437a36:46291 {}] regionserver.HRegionServer(976): stopping server 1cb9cf437a36,46291,1732777007811; all regions closed. 2024-11-28T06:56:51,192 DEBUG [RS:1;1cb9cf437a36:46291 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7e9a7384-d065-9ff8-196a-59688d3a4920/752619e7-664e-6a55-3bd6-93cba9c9ba92/oldWALs 2024-11-28T06:56:51,193 INFO [RS:1;1cb9cf437a36:46291 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 1cb9cf437a36%2C46291%2C1732777007811.meta:.meta(num 1732777009089) 2024-11-28T06:56:51,194 DEBUG [RS:1;1cb9cf437a36:46291 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7e9a7384-d065-9ff8-196a-59688d3a4920/752619e7-664e-6a55-3bd6-93cba9c9ba92/oldWALs 2024-11-28T06:56:51,194 INFO [RS:1;1cb9cf437a36:46291 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 1cb9cf437a36%2C46291%2C1732777007811:(num 1732777008670) 2024-11-28T06:56:51,194 DEBUG [RS:1;1cb9cf437a36:46291 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T06:56:51,194 INFO [RS:1;1cb9cf437a36:46291 {}] regionserver.LeaseManager(133): Closed leases 2024-11-28T06:56:51,194 INFO [RS:1;1cb9cf437a36:46291 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-28T06:56:51,195 INFO [RS:1;1cb9cf437a36:46291 {}] hbase.ChoreService(370): Chore service for: regionserver/1cb9cf437a36:0 had [ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-11-28T06:56:51,195 INFO [RS:1;1cb9cf437a36:46291 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-28T06:56:51,195 INFO [regionserver/1cb9cf437a36:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-28T06:56:51,195 INFO [RS:1;1cb9cf437a36:46291 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:46291 2024-11-28T06:56:51,221 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42945-0x10181846c940000, quorum=127.0.0.1:61052, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-28T06:56:51,221 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46291-0x10181846c940002, quorum=127.0.0.1:61052, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/1cb9cf437a36,46291,1732777007811 2024-11-28T06:56:51,221 INFO [RS:1;1cb9cf437a36:46291 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-28T06:56:51,229 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [1cb9cf437a36,46291,1732777007811] 2024-11-28T06:56:51,237 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/1cb9cf437a36,46291,1732777007811 already deleted, retry=false 2024-11-28T06:56:51,237 INFO [RegionServerTracker-0 {}] master.ServerManager(695): Processing expiration of 1cb9cf437a36,46291,1732777007811 on 1cb9cf437a36,42945,1732777007591 2024-11-28T06:56:51,243 DEBUG [RegionServerTracker-0 {}] procedure2.ProcedureExecutor(1139): Stored pid=7, state=RUNNABLE:SERVER_CRASH_START, hasLock=false; ServerCrashProcedure 1cb9cf437a36,46291,1732777007811, splitWal=true, meta=true 2024-11-28T06:56:51,247 INFO [PEWorker-2 {}] procedure.ServerCrashProcedure(169): Start pid=7, state=RUNNABLE:SERVER_CRASH_START, hasLock=true; ServerCrashProcedure 1cb9cf437a36,46291,1732777007811, splitWal=true, meta=true 2024-11-28T06:56:51,247 INFO [RegionServerTracker-0 {}] assignment.AssignmentManager(1999): Scheduled ServerCrashProcedure pid=7 for 1cb9cf437a36,46291,1732777007811 (carryingMeta=true) 1cb9cf437a36,46291,1732777007811/CRASHED/regionCount=2/lock=java.util.concurrent.locks.ReentrantReadWriteLock@765d0703[Write locks = 1, Read locks = 0], oldState=ONLINE. 2024-11-28T06:56:51,249 INFO [PEWorker-2 {}] procedure.ServerCrashProcedure(339): Splitting WALs pid=7, state=RUNNABLE:SERVER_CRASH_SPLIT_META_LOGS, hasLock=true; ServerCrashProcedure 1cb9cf437a36,46291,1732777007811, splitWal=true, meta=true, isMeta: true 2024-11-28T06:56:51,250 DEBUG [PEWorker-2 {}] master.MasterWalManager(329): Renamed region directory: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7e9a7384-d065-9ff8-196a-59688d3a4920/752619e7-664e-6a55-3bd6-93cba9c9ba92/WALs/1cb9cf437a36,46291,1732777007811-splitting 2024-11-28T06:56:51,250 INFO [PEWorker-2 {}] master.SplitLogManager(171): file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7e9a7384-d065-9ff8-196a-59688d3a4920/752619e7-664e-6a55-3bd6-93cba9c9ba92/WALs/1cb9cf437a36,46291,1732777007811-splitting dir is empty, no logs to split. 2024-11-28T06:56:51,250 INFO [PEWorker-2 {}] master.SplitWALManager(105): 1cb9cf437a36,46291,1732777007811 WAL count=0, meta=true 2024-11-28T06:56:51,252 INFO [PEWorker-2 {}] master.SplitLogManager(171): file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7e9a7384-d065-9ff8-196a-59688d3a4920/752619e7-664e-6a55-3bd6-93cba9c9ba92/WALs/1cb9cf437a36,46291,1732777007811-splitting dir is empty, no logs to split. 2024-11-28T06:56:51,252 INFO [PEWorker-2 {}] master.SplitWALManager(105): 1cb9cf437a36,46291,1732777007811 WAL count=0, meta=true 2024-11-28T06:56:51,252 DEBUG [PEWorker-2 {}] procedure.ServerCrashProcedure(329): Check if 1cb9cf437a36,46291,1732777007811 WAL splitting is done? wals=0, meta=true 2024-11-28T06:56:51,254 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=8, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-11-28T06:56:51,255 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=8, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-28T06:56:51,255 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(269): Starting pid=8, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-11-28T06:56:51,330 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46291-0x10181846c940002, quorum=127.0.0.1:61052, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-28T06:56:51,330 INFO [RS:1;1cb9cf437a36:46291 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-28T06:56:51,330 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46291-0x10181846c940002, quorum=127.0.0.1:61052, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-28T06:56:51,330 INFO [RS:1;1cb9cf437a36:46291 {}] regionserver.HRegionServer(1031): Exiting; stopping=1cb9cf437a36,46291,1732777007811; zookeeper connection closed. 2024-11-28T06:56:51,331 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@27d11ab2 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@27d11ab2 2024-11-28T06:56:51,406 DEBUG [1cb9cf437a36:42945 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=1, allServersCount=1 2024-11-28T06:56:51,407 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=8 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=1cb9cf437a36,40661,1732777007755 2024-11-28T06:56:51,409 INFO [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 1cb9cf437a36,40661,1732777007755, state=OPENING 2024-11-28T06:56:51,433 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40661-0x10181846c940001, quorum=127.0.0.1:61052, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-28T06:56:51,433 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42945-0x10181846c940000, quorum=127.0.0.1:61052, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-28T06:56:51,433 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=8, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-28T06:56:51,433 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-28T06:56:51,433 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-28T06:56:51,433 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=9, ppid=8, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=1cb9cf437a36,40661,1732777007755}] 2024-11-28T06:56:51,493 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/1cb9cf437a36:0 server-side Connection retries=45 2024-11-28T06:56:51,493 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-28T06:56:51,494 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-28T06:56:51,494 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-28T06:56:51,494 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-28T06:56:51,494 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-28T06:56:51,494 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-28T06:56:51,494 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-28T06:56:51,495 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:39675 2024-11-28T06:56:51,496 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:39675 connecting to ZooKeeper ensemble=127.0.0.1:61052 2024-11-28T06:56:51,496 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-28T06:56:51,497 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-28T06:56:51,521 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:396750x0, quorum=127.0.0.1:61052, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-28T06:56:51,521 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:39675-0x10181846c940003 connected 2024-11-28T06:56:51,521 DEBUG [Time-limited test {}] zookeeper.ZKUtil(111): regionserver:39675-0x10181846c940003, quorum=127.0.0.1:61052, baseZNode=/hbase Set watcher on existing znode=/hbase/running 2024-11-28T06:56:51,521 DEBUG [pool-298-thread-1 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: INIT 2024-11-28T06:56:51,522 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-28T06:56:51,523 INFO [Time-limited test {}] bucket.FileIOEngine(99): Allocating cache 400 MB, on the path:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/33d1270a-7ecb-879e-376a-4fd885e9a050/bucket.cache 2024-11-28T06:56:51,523 INFO [Time-limited test {}] bucket.BucketCache(364): Instantiating BucketCache with acceptableFactor: 0.95, minFactor: 0.85, extraFreeFactor: 0.1, singleFactor: 0.25, multiFactor: 0.5, memoryFactor: 0.25, useStrongRef: false 2024-11-28T06:56:51,523 INFO [Time-limited test {}] bucket.BucketCache(478): Starting BucketCachePersisterThread 2024-11-28T06:56:51,523 INFO [Time-limited test {}] bucket.BucketCachePersister(38): BucketCachePersister started with interval: 100 2024-11-28T06:56:51,525 INFO [Thread-207 {}] bucket.BucketCache(1530): Started retrieving bucket cache from file 2024-11-28T06:56:51,526 INFO [Time-limited test {}] bucket.BucketCache(403): Started bucket cache; ioengine=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/33d1270a-7ecb-879e-376a-4fd885e9a050/bucket.cache, capacity=400 MB, blockSize=64 KB, writerThreadNum=3, writerQLen=64, persistencePath=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/33d1270a-7ecb-879e-376a-4fd885e9a050/bucket.persistence, bucketAllocator=org.apache.hadoop.hbase.io.hfile.bucket.BucketAllocator 2024-11-28T06:56:51,526 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-28T06:56:51,527 INFO [Thread-207 {}] bucket.BucketCache(1552): Reading new chunked format of persistence. 2024-11-28T06:56:51,527 DEBUG [Time-limited test {}] zookeeper.ZKUtil(111): regionserver:39675-0x10181846c940003, quorum=127.0.0.1:61052, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-11-28T06:56:51,528 INFO [Thread-207 {}] bucket.BucketCache(1705): Retrieved 1 of chunks with blockCount = 1. 2024-11-28T06:56:51,530 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:39675-0x10181846c940003, quorum=127.0.0.1:61052, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-28T06:56:51,533 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=39675 2024-11-28T06:56:51,533 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=39675 2024-11-28T06:56:51,533 DEBUG [Thread-207 {}] bucket.BucketCache(1569): Updating region size map after retrieving cached file list 2024-11-28T06:56:51,533 DEBUG [Thread-207 {}] bucket.BucketCache(1584): Cached File Entry:<68fcf27268094562965c0951b6ee0334,<31ba9c474ceabbeaab02529db668ff6e,158>> 2024-11-28T06:56:51,534 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=39675 2024-11-28T06:56:51,535 INFO [Thread-207 {}] bucket.BucketAllocator(343): Cache totalSize=418148352, buckets=199, bucket capacity=2101248=(4*525312)=(FEWEST_ITEMS_IN_BUCKET*(largest configured bucketcache size)) 2024-11-28T06:56:51,535 INFO [Thread-207 {}] bucket.BucketCache(1562): Bucket cache retrieved from file successfully with size: 1 2024-11-28T06:56:51,535 INFO [Thread-207 {}] bucket.BucketCache(413): Persistent bucket cache recovery from /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/33d1270a-7ecb-879e-376a-4fd885e9a050/bucket.persistence is complete. 2024-11-28T06:56:51,535 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=39675 2024-11-28T06:56:51,538 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=39675 2024-11-28T06:56:51,540 INFO [RS:2;1cb9cf437a36:39675 {}] regionserver.HRegionServer(746): ClusterId : f1e003c1-57de-44c5-9f50-31707ec95057 2024-11-28T06:56:51,540 DEBUG [RS:2;1cb9cf437a36:39675 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-28T06:56:51,550 DEBUG [RS:2;1cb9cf437a36:39675 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-28T06:56:51,551 DEBUG [RS:2;1cb9cf437a36:39675 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-28T06:56:51,559 DEBUG [RS:2;1cb9cf437a36:39675 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-28T06:56:51,559 DEBUG [RS:2;1cb9cf437a36:39675 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1f2e2f93, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=1cb9cf437a36/172.17.0.2:0 2024-11-28T06:56:51,573 DEBUG [RS:2;1cb9cf437a36:39675 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:2;1cb9cf437a36:39675 2024-11-28T06:56:51,573 INFO [RS:2;1cb9cf437a36:39675 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-28T06:56:51,574 INFO [RS:2;1cb9cf437a36:39675 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-28T06:56:51,574 DEBUG [RS:2;1cb9cf437a36:39675 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-28T06:56:51,574 INFO [RS:2;1cb9cf437a36:39675 {}] regionserver.HRegionServer(2659): reportForDuty to master=1cb9cf437a36,42945,1732777007591 with port=39675, startcode=1732777011493 2024-11-28T06:56:51,575 DEBUG [RS:2;1cb9cf437a36:39675 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-28T06:56:51,576 INFO [HMaster-EventLoopGroup-6-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59315, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=RegionServerStatusService 2024-11-28T06:56:51,577 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=42945 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 1cb9cf437a36,39675,1732777011493 2024-11-28T06:56:51,577 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=42945 {}] master.ServerManager(517): Registering regionserver=1cb9cf437a36,39675,1732777011493 2024-11-28T06:56:51,578 DEBUG [RS:2;1cb9cf437a36:39675 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7e9a7384-d065-9ff8-196a-59688d3a4920/752619e7-664e-6a55-3bd6-93cba9c9ba92 2024-11-28T06:56:51,578 DEBUG [RS:2;1cb9cf437a36:39675 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=file:/ 2024-11-28T06:56:51,578 DEBUG [RS:2;1cb9cf437a36:39675 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-28T06:56:51,587 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42945-0x10181846c940000, quorum=127.0.0.1:61052, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-28T06:56:51,588 DEBUG [RS:2;1cb9cf437a36:39675 {}] zookeeper.ZKUtil(111): regionserver:39675-0x10181846c940003, quorum=127.0.0.1:61052, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/1cb9cf437a36,39675,1732777011493 2024-11-28T06:56:51,588 WARN [RS:2;1cb9cf437a36:39675 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-28T06:56:51,588 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [1cb9cf437a36,39675,1732777011493] 2024-11-28T06:56:51,588 INFO [RS:2;1cb9cf437a36:39675 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-28T06:56:51,588 DEBUG [RS:2;1cb9cf437a36:39675 {}] regionserver.HRegionServer(1793): logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7e9a7384-d065-9ff8-196a-59688d3a4920/752619e7-664e-6a55-3bd6-93cba9c9ba92/WALs/1cb9cf437a36,39675,1732777011493 2024-11-28T06:56:51,588 DEBUG [RSProcedureDispatcher-pool-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-28T06:56:51,591 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-7-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35413, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-28T06:56:51,593 INFO [RS_OPEN_META-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_META, pid=9}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-11-28T06:56:51,593 INFO [RS_OPEN_META-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_META, pid=9}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-28T06:56:51,593 INFO [RS:2;1cb9cf437a36:39675 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-28T06:56:51,594 INFO [RS_OPEN_META-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_META, pid=9}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-11-28T06:56:51,595 INFO [RS_OPEN_META-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_META, pid=9}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=1cb9cf437a36%2C40661%2C1732777007755.meta, suffix=.meta, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7e9a7384-d065-9ff8-196a-59688d3a4920/752619e7-664e-6a55-3bd6-93cba9c9ba92/WALs/1cb9cf437a36,40661,1732777007755, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7e9a7384-d065-9ff8-196a-59688d3a4920/752619e7-664e-6a55-3bd6-93cba9c9ba92/oldWALs, maxLogs=55 2024-11-28T06:56:51,597 INFO [RS:2;1cb9cf437a36:39675 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-28T06:56:51,597 INFO [RS:2;1cb9cf437a36:39675 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-28T06:56:51,597 INFO [RS:2;1cb9cf437a36:39675 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-28T06:56:51,601 INFO [RS:2;1cb9cf437a36:39675 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-28T06:56:51,601 INFO [RS:2;1cb9cf437a36:39675 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-28T06:56:51,601 INFO [RS:2;1cb9cf437a36:39675 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-28T06:56:51,602 DEBUG [RS:2;1cb9cf437a36:39675 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/1cb9cf437a36:0, corePoolSize=1, maxPoolSize=1 2024-11-28T06:56:51,602 DEBUG [RS:2;1cb9cf437a36:39675 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/1cb9cf437a36:0, corePoolSize=1, maxPoolSize=1 2024-11-28T06:56:51,602 DEBUG [RS:2;1cb9cf437a36:39675 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/1cb9cf437a36:0, corePoolSize=1, maxPoolSize=1 2024-11-28T06:56:51,602 DEBUG [RS:2;1cb9cf437a36:39675 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/1cb9cf437a36:0, corePoolSize=1, maxPoolSize=1 2024-11-28T06:56:51,602 DEBUG [RS:2;1cb9cf437a36:39675 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/1cb9cf437a36:0, corePoolSize=1, maxPoolSize=1 2024-11-28T06:56:51,602 DEBUG [RS:2;1cb9cf437a36:39675 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/1cb9cf437a36:0, corePoolSize=2, maxPoolSize=2 2024-11-28T06:56:51,602 DEBUG [RS:2;1cb9cf437a36:39675 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/1cb9cf437a36:0, corePoolSize=1, maxPoolSize=1 2024-11-28T06:56:51,602 DEBUG [RS:2;1cb9cf437a36:39675 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/1cb9cf437a36:0, corePoolSize=1, maxPoolSize=1 2024-11-28T06:56:51,602 DEBUG [RS:2;1cb9cf437a36:39675 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/1cb9cf437a36:0, corePoolSize=1, maxPoolSize=1 2024-11-28T06:56:51,602 DEBUG [RS:2;1cb9cf437a36:39675 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/1cb9cf437a36:0, corePoolSize=1, maxPoolSize=1 2024-11-28T06:56:51,603 DEBUG [RS:2;1cb9cf437a36:39675 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/1cb9cf437a36:0, corePoolSize=1, maxPoolSize=1 2024-11-28T06:56:51,603 DEBUG [RS:2;1cb9cf437a36:39675 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/1cb9cf437a36:0, corePoolSize=1, maxPoolSize=1 2024-11-28T06:56:51,603 DEBUG [RS:2;1cb9cf437a36:39675 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/1cb9cf437a36:0, corePoolSize=3, maxPoolSize=3 2024-11-28T06:56:51,603 DEBUG [RS:2;1cb9cf437a36:39675 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/1cb9cf437a36:0, corePoolSize=3, maxPoolSize=3 2024-11-28T06:56:51,603 INFO [RS:2;1cb9cf437a36:39675 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-28T06:56:51,603 INFO [RS:2;1cb9cf437a36:39675 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-28T06:56:51,603 INFO [RS:2;1cb9cf437a36:39675 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-28T06:56:51,603 INFO [RS:2;1cb9cf437a36:39675 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-28T06:56:51,604 INFO [RS:2;1cb9cf437a36:39675 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-28T06:56:51,604 INFO [RS:2;1cb9cf437a36:39675 {}] hbase.ChoreService(168): Chore ScheduledChore name=1cb9cf437a36,39675,1732777011493-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-28T06:56:51,614 INFO [RS_OPEN_META-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_META, pid=9}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7e9a7384-d065-9ff8-196a-59688d3a4920/752619e7-664e-6a55-3bd6-93cba9c9ba92/WALs/1cb9cf437a36,40661,1732777007755/1cb9cf437a36%2C40661%2C1732777007755.meta.1732777011596.meta 2024-11-28T06:56:51,617 DEBUG [RS_OPEN_META-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_META, pid=9}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-11-28T06:56:51,617 DEBUG [RS_OPEN_META-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-11-28T06:56:51,618 DEBUG [RS_OPEN_META-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_META, pid=9}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-11-28T06:56:51,618 DEBUG [RS_OPEN_META-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-11-28T06:56:51,618 INFO [RS_OPEN_META-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-11-28T06:56:51,618 DEBUG [RS_OPEN_META-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-11-28T06:56:51,618 DEBUG [RS_OPEN_META-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-28T06:56:51,618 DEBUG [RS_OPEN_META-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-11-28T06:56:51,618 DEBUG [RS_OPEN_META-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-11-28T06:56:51,619 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family info of region 1588230740 2024-11-28T06:56:51,619 INFO [RS:2;1cb9cf437a36:39675 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-28T06:56:51,619 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-28T06:56:51,620 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:56:51,620 INFO [RS:2;1cb9cf437a36:39675 {}] hbase.ChoreService(168): Chore ScheduledChore name=1cb9cf437a36,39675,1732777011493-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-28T06:56:51,620 INFO [RS:2;1cb9cf437a36:39675 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-28T06:56:51,620 INFO [RS:2;1cb9cf437a36:39675 {}] regionserver.Replication(171): 1cb9cf437a36,39675,1732777011493 started 2024-11-28T06:56:51,621 DEBUG [StoreFileOpener-1588230740-info-1 {}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7e9a7384-d065-9ff8-196a-59688d3a4920/752619e7-664e-6a55-3bd6-93cba9c9ba92/data/hbase/meta/1588230740/info/666397d4cded4a028775335b64d1e84e, delay=903 ms 2024-11-28T06:56:51,622 DEBUG [StoreOpener-1588230740-1 {}] regionserver.StoreEngine(278): loaded file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7e9a7384-d065-9ff8-196a-59688d3a4920/752619e7-664e-6a55-3bd6-93cba9c9ba92/data/hbase/meta/1588230740/info/666397d4cded4a028775335b64d1e84e 2024-11-28T06:56:51,622 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-28T06:56:51,622 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family ns of region 1588230740 2024-11-28T06:56:51,623 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-28T06:56:51,623 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:56:51,624 DEBUG [StoreFileOpener-1588230740-ns-1 {}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7e9a7384-d065-9ff8-196a-59688d3a4920/752619e7-664e-6a55-3bd6-93cba9c9ba92/data/hbase/meta/1588230740/ns/4ef821c80f9a4e05ab163ae8a76012e3, delay=954 ms 2024-11-28T06:56:51,625 DEBUG [StoreOpener-1588230740-1 {}] regionserver.StoreEngine(278): loaded file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7e9a7384-d065-9ff8-196a-59688d3a4920/752619e7-664e-6a55-3bd6-93cba9c9ba92/data/hbase/meta/1588230740/ns/4ef821c80f9a4e05ab163ae8a76012e3 2024-11-28T06:56:51,626 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-28T06:56:51,626 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family rep_barrier of region 1588230740 2024-11-28T06:56:51,626 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-28T06:56:51,626 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:56:51,626 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-28T06:56:51,627 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family table of region 1588230740 2024-11-28T06:56:51,627 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-28T06:56:51,627 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:56:51,628 DEBUG [StoreFileOpener-1588230740-table-1 {}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7e9a7384-d065-9ff8-196a-59688d3a4920/752619e7-664e-6a55-3bd6-93cba9c9ba92/data/hbase/meta/1588230740/table/9b2097e829dc4962a8584e9c1d2359d8, delay=976 ms 2024-11-28T06:56:51,629 DEBUG [StoreOpener-1588230740-1 {}] regionserver.StoreEngine(278): loaded file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7e9a7384-d065-9ff8-196a-59688d3a4920/752619e7-664e-6a55-3bd6-93cba9c9ba92/data/hbase/meta/1588230740/table/9b2097e829dc4962a8584e9c1d2359d8 2024-11-28T06:56:51,629 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-28T06:56:51,630 DEBUG [RS_OPEN_META-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-28T06:56:51,630 DEBUG [RS_OPEN_META-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7e9a7384-d065-9ff8-196a-59688d3a4920/752619e7-664e-6a55-3bd6-93cba9c9ba92/data/hbase/meta/1588230740 2024-11-28T06:56:51,630 DEBUG [RS_OPEN_META-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7e9a7384-d065-9ff8-196a-59688d3a4920/752619e7-664e-6a55-3bd6-93cba9c9ba92/data/hbase/meta/1588230740 2024-11-28T06:56:51,631 DEBUG [RS_OPEN_META-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-28T06:56:51,631 DEBUG [RS_OPEN_META-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-28T06:56:51,631 DEBUG [RS_OPEN_META-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-28T06:56:51,632 DEBUG [RS_OPEN_META-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-28T06:56:51,632 INFO [RS_OPEN_META-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=15; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=60027559, jitterRate=-0.10551966726779938}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-28T06:56:51,632 DEBUG [RS_OPEN_META-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-11-28T06:56:51,633 DEBUG [RS_OPEN_META-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1732777011618Writing region info on filesystem at 1732777011618Initializing all the Stores at 1732777011619 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732777011619Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732777011619Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732777011619Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732777011619Cleaning up temporary data from old regions at 1732777011631 (+12 ms)Running coprocessor post-open hooks at 1732777011633 (+2 ms)Region opened successfully at 1732777011633 2024-11-28T06:56:51,634 INFO [RS_OPEN_META-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=9, masterSystemTime=1732777011588 2024-11-28T06:56:51,637 DEBUG [RS_OPEN_META-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-11-28T06:56:51,637 INFO [RS_OPEN_META-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_META, pid=9}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-11-28T06:56:51,637 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=8 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=15, regionLocation=1cb9cf437a36,40661,1732777007755 2024-11-28T06:56:51,638 INFO [PEWorker-1 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 1cb9cf437a36,40661,1732777007755, state=OPEN 2024-11-28T06:56:51,639 INFO [RS:2;1cb9cf437a36:39675 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-28T06:56:51,639 INFO [RS:2;1cb9cf437a36:39675 {}] regionserver.HRegionServer(1482): Serving as 1cb9cf437a36,39675,1732777011493, RpcServer on 1cb9cf437a36/172.17.0.2:39675, sessionid=0x10181846c940003 2024-11-28T06:56:51,640 DEBUG [RS:2;1cb9cf437a36:39675 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-28T06:56:51,640 DEBUG [RS:2;1cb9cf437a36:39675 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 1cb9cf437a36,39675,1732777011493 2024-11-28T06:56:51,640 DEBUG [RS:2;1cb9cf437a36:39675 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '1cb9cf437a36,39675,1732777011493' 2024-11-28T06:56:51,640 DEBUG [RS:2;1cb9cf437a36:39675 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-28T06:56:51,658 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40661-0x10181846c940001, quorum=127.0.0.1:61052, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-28T06:56:51,658 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42945-0x10181846c940000, quorum=127.0.0.1:61052, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-28T06:56:51,658 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39675-0x10181846c940003, quorum=127.0.0.1:61052, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-28T06:56:51,658 DEBUG [RS:2;1cb9cf437a36:39675 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-28T06:56:51,658 DEBUG [PEWorker-1 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=9, ppid=8, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=1cb9cf437a36,40661,1732777007755 2024-11-28T06:56:51,658 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-28T06:56:51,658 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-28T06:56:51,658 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-28T06:56:51,659 DEBUG [RS:2;1cb9cf437a36:39675 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-28T06:56:51,659 DEBUG [RS:2;1cb9cf437a36:39675 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-28T06:56:51,659 DEBUG [RS:2;1cb9cf437a36:39675 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 1cb9cf437a36,39675,1732777011493 2024-11-28T06:56:51,659 DEBUG [RS:2;1cb9cf437a36:39675 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '1cb9cf437a36,39675,1732777011493' 2024-11-28T06:56:51,659 DEBUG [RS:2;1cb9cf437a36:39675 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-28T06:56:51,659 DEBUG [RS:2;1cb9cf437a36:39675 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-28T06:56:51,660 DEBUG [RS:2;1cb9cf437a36:39675 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-28T06:56:51,660 INFO [RS:2;1cb9cf437a36:39675 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-28T06:56:51,660 INFO [RS:2;1cb9cf437a36:39675 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-28T06:56:51,660 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=9, resume processing ppid=8 2024-11-28T06:56:51,660 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=9, ppid=8, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=1cb9cf437a36,40661,1732777007755 in 225 msec 2024-11-28T06:56:51,662 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=8, resume processing ppid=7 2024-11-28T06:56:51,662 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=8, ppid=7, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 406 msec 2024-11-28T06:56:51,662 INFO [PEWorker-3 {}] procedure.ServerCrashProcedure(207): 1cb9cf437a36,46291,1732777007811 had 2 regions 2024-11-28T06:56:51,663 INFO [PEWorker-3 {}] procedure.ServerCrashProcedure(339): Splitting WALs pid=7, state=RUNNABLE:SERVER_CRASH_SPLIT_LOGS, hasLock=true; ServerCrashProcedure 1cb9cf437a36,46291,1732777007811, splitWal=true, meta=true, isMeta: false 2024-11-28T06:56:51,664 INFO [PEWorker-3 {}] master.SplitLogManager(171): file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7e9a7384-d065-9ff8-196a-59688d3a4920/752619e7-664e-6a55-3bd6-93cba9c9ba92/WALs/1cb9cf437a36,46291,1732777007811-splitting dir is empty, no logs to split. 2024-11-28T06:56:51,664 INFO [PEWorker-3 {}] master.SplitWALManager(105): 1cb9cf437a36,46291,1732777007811 WAL count=0, meta=false 2024-11-28T06:56:51,665 INFO [PEWorker-3 {}] master.SplitLogManager(171): file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7e9a7384-d065-9ff8-196a-59688d3a4920/752619e7-664e-6a55-3bd6-93cba9c9ba92/WALs/1cb9cf437a36,46291,1732777007811-splitting dir is empty, no logs to split. 2024-11-28T06:56:51,665 INFO [PEWorker-3 {}] master.SplitWALManager(105): 1cb9cf437a36,46291,1732777007811 WAL count=0, meta=false 2024-11-28T06:56:51,665 DEBUG [PEWorker-3 {}] procedure.ServerCrashProcedure(329): Check if 1cb9cf437a36,46291,1732777007811 WAL splitting is done? wals=0, meta=false 2024-11-28T06:56:51,666 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=10, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1732777009287, region=31ba9c474ceabbeaab02529db668ff6e, ASSIGN}] 2024-11-28T06:56:51,667 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=10, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1732777009287, region=31ba9c474ceabbeaab02529db668ff6e, ASSIGN 2024-11-28T06:56:51,668 INFO [PEWorker-5 {}] assignment.TransitRegionStateProcedure(269): Starting pid=10, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1732777009287, region=31ba9c474ceabbeaab02529db668ff6e, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-11-28T06:56:51,761 INFO [RS:2;1cb9cf437a36:39675 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-28T06:56:51,761 INFO [RS:2;1cb9cf437a36:39675 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=1cb9cf437a36%2C39675%2C1732777011493, suffix=, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7e9a7384-d065-9ff8-196a-59688d3a4920/752619e7-664e-6a55-3bd6-93cba9c9ba92/WALs/1cb9cf437a36,39675,1732777011493, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7e9a7384-d065-9ff8-196a-59688d3a4920/752619e7-664e-6a55-3bd6-93cba9c9ba92/oldWALs, maxLogs=55 2024-11-28T06:56:51,782 INFO [RS:2;1cb9cf437a36:39675 {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7e9a7384-d065-9ff8-196a-59688d3a4920/752619e7-664e-6a55-3bd6-93cba9c9ba92/WALs/1cb9cf437a36,39675,1732777011493/1cb9cf437a36%2C39675%2C1732777011493.1732777011762 2024-11-28T06:56:51,782 DEBUG [RS:2;1cb9cf437a36:39675 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-11-28T06:56:51,818 DEBUG [1cb9cf437a36:42945 {}] balancer.BalancerClusterState(204): Hosts are {1cb9cf437a36=0} racks are {/default-rack=0} 2024-11-28T06:56:51,819 DEBUG [1cb9cf437a36:42945 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-11-28T06:56:51,819 DEBUG [1cb9cf437a36:42945 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-11-28T06:56:51,819 DEBUG [1cb9cf437a36:42945 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-11-28T06:56:51,819 DEBUG [1cb9cf437a36:42945 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-11-28T06:56:51,819 INFO [1cb9cf437a36:42945 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-11-28T06:56:51,819 INFO [1cb9cf437a36:42945 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-11-28T06:56:51,819 DEBUG [1cb9cf437a36:42945 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-11-28T06:56:51,819 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=10 updating hbase:meta row=31ba9c474ceabbeaab02529db668ff6e, regionState=OPENING, regionLocation=1cb9cf437a36,40661,1732777007755 2024-11-28T06:56:51,820 WARN [MiniHBaseClusterRegionServer-EventLoopGroup-8-3 {}] ipc.NettyRpcConnection$2(409): Exception encountered while connecting to the server 1cb9cf437a36:46291 org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 1cb9cf437a36/172.17.0.2:46291 Caused by: java.net.ConnectException: finishConnect(..) failed: Connection refused at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.newConnectException0(Errors.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.handleConnectErrno(Errors.java:131) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Socket.finishConnect(Socket.java:359) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.doFinishConnect(AbstractEpollChannel.java:715) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:692) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T06:56:51,822 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-8-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=1cb9cf437a36,46291,1732777007811, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=1cb9cf437a36,46291,1732777007811, seqNum=-1, error=java.net.ConnectException: Call to address=1cb9cf437a36:46291 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 1cb9cf437a36/172.17.0.2:46291 2024-11-28T06:56:51,823 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-8-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:meta,,1.1588230740, hostname=1cb9cf437a36,46291,1732777007811, seqNum=-1 is java.net.ConnectException: finishConnect(..) failed: Connection refused 2024-11-28T06:56:51,823 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-8-3 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:meta,,1.1588230740, hostname=1cb9cf437a36,46291,1732777007811, seqNum=-1 from cache 2024-11-28T06:56:51,823 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-8-3 {}] ipc.FailedServers(52): Added failed server with address 1cb9cf437a36:46291 to list caused by org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 1cb9cf437a36/172.17.0.2:46291 2024-11-28T06:56:51,931 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-28T06:56:51,931 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=1cb9cf437a36,40661,1732777007755, seqNum=-1] 2024-11-28T06:56:51,932 DEBUG [Async-Client-Retry-Timer-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-28T06:56:51,935 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-7-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:55785, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-28T06:56:51,940 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-8-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=10, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1732777009287, region=31ba9c474ceabbeaab02529db668ff6e, ASSIGN because future has completed 2024-11-28T06:56:51,941 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=11, ppid=10, state=RUNNABLE, hasLock=false; OpenRegionProcedure 31ba9c474ceabbeaab02529db668ff6e, server=1cb9cf437a36,40661,1732777007755}] 2024-11-28T06:56:52,100 INFO [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] handler.AssignRegionHandler(132): Open testBlockEvictionOnGracefulStop1732777009287,,1732777009293.31ba9c474ceabbeaab02529db668ff6e. 2024-11-28T06:56:52,101 DEBUG [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(7752): Opening region: {ENCODED => 31ba9c474ceabbeaab02529db668ff6e, NAME => 'testBlockEvictionOnGracefulStop1732777009287,,1732777009293.31ba9c474ceabbeaab02529db668ff6e.', STARTKEY => '', ENDKEY => ''} 2024-11-28T06:56:52,102 DEBUG [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testBlockEvictionOnGracefulStop1732777009287 31ba9c474ceabbeaab02529db668ff6e 2024-11-28T06:56:52,102 DEBUG [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(898): Instantiated testBlockEvictionOnGracefulStop1732777009287,,1732777009293.31ba9c474ceabbeaab02529db668ff6e.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-28T06:56:52,102 DEBUG [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(7794): checking encryption for 31ba9c474ceabbeaab02529db668ff6e 2024-11-28T06:56:52,102 DEBUG [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(7797): checking classloading for 31ba9c474ceabbeaab02529db668ff6e 2024-11-28T06:56:52,103 INFO [StoreOpener-31ba9c474ceabbeaab02529db668ff6e-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family family of region 31ba9c474ceabbeaab02529db668ff6e 2024-11-28T06:56:52,103 INFO [StoreOpener-31ba9c474ceabbeaab02529db668ff6e-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 31ba9c474ceabbeaab02529db668ff6e columnFamilyName family 2024-11-28T06:56:52,103 DEBUG [StoreOpener-31ba9c474ceabbeaab02529db668ff6e-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T06:56:52,105 DEBUG [StoreFileOpener-31ba9c474ceabbeaab02529db668ff6e-family-1 {}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7e9a7384-d065-9ff8-196a-59688d3a4920/752619e7-664e-6a55-3bd6-93cba9c9ba92/data/default/testBlockEvictionOnGracefulStop1732777009287/31ba9c474ceabbeaab02529db668ff6e/family/68fcf27268094562965c0951b6ee0334, delay=938 ms 2024-11-28T06:56:52,106 DEBUG [StoreOpener-31ba9c474ceabbeaab02529db668ff6e-1 {}] regionserver.StoreEngine(278): loaded file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7e9a7384-d065-9ff8-196a-59688d3a4920/752619e7-664e-6a55-3bd6-93cba9c9ba92/data/default/testBlockEvictionOnGracefulStop1732777009287/31ba9c474ceabbeaab02529db668ff6e/family/68fcf27268094562965c0951b6ee0334 2024-11-28T06:56:52,106 INFO [StoreOpener-31ba9c474ceabbeaab02529db668ff6e-1 {}] regionserver.HStore(327): Store=31ba9c474ceabbeaab02529db668ff6e/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-28T06:56:52,106 DEBUG [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(1038): replaying wal for 31ba9c474ceabbeaab02529db668ff6e 2024-11-28T06:56:52,107 DEBUG [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7e9a7384-d065-9ff8-196a-59688d3a4920/752619e7-664e-6a55-3bd6-93cba9c9ba92/data/default/testBlockEvictionOnGracefulStop1732777009287/31ba9c474ceabbeaab02529db668ff6e 2024-11-28T06:56:52,107 DEBUG [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7e9a7384-d065-9ff8-196a-59688d3a4920/752619e7-664e-6a55-3bd6-93cba9c9ba92/data/default/testBlockEvictionOnGracefulStop1732777009287/31ba9c474ceabbeaab02529db668ff6e 2024-11-28T06:56:52,108 DEBUG [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(1048): stopping wal replay for 31ba9c474ceabbeaab02529db668ff6e 2024-11-28T06:56:52,108 DEBUG [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(1060): Cleaning up temporary data for 31ba9c474ceabbeaab02529db668ff6e 2024-11-28T06:56:52,109 DEBUG [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(1093): writing seq id for 31ba9c474ceabbeaab02529db668ff6e 2024-11-28T06:56:52,109 INFO [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(1114): Opened 31ba9c474ceabbeaab02529db668ff6e; next sequenceid=10; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=71251502, jitterRate=0.06173011660575867}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-28T06:56:52,109 DEBUG [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 31ba9c474ceabbeaab02529db668ff6e 2024-11-28T06:56:52,110 DEBUG [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(1006): Region open journal for 31ba9c474ceabbeaab02529db668ff6e: Running coprocessor pre-open hook at 1732777012102Writing region info on filesystem at 1732777012102Initializing all the Stores at 1732777012102Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732777012102Cleaning up temporary data from old regions at 1732777012108 (+6 ms)Running coprocessor post-open hooks at 1732777012109 (+1 ms)Region opened successfully at 1732777012110 (+1 ms) 2024-11-28T06:56:52,110 INFO [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegionServer(2236): Post open deploy tasks for testBlockEvictionOnGracefulStop1732777009287,,1732777009293.31ba9c474ceabbeaab02529db668ff6e., pid=11, masterSystemTime=1732777012096 2024-11-28T06:56:52,113 DEBUG [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegionServer(2266): Finished post open deploy task for testBlockEvictionOnGracefulStop1732777009287,,1732777009293.31ba9c474ceabbeaab02529db668ff6e. 2024-11-28T06:56:52,113 INFO [RS_OPEN_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] handler.AssignRegionHandler(153): Opened testBlockEvictionOnGracefulStop1732777009287,,1732777009293.31ba9c474ceabbeaab02529db668ff6e. 2024-11-28T06:56:52,114 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=10 updating hbase:meta row=31ba9c474ceabbeaab02529db668ff6e, regionState=OPEN, openSeqNum=10, regionLocation=1cb9cf437a36,40661,1732777007755 2024-11-28T06:56:52,116 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-8-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=11, ppid=10, state=RUNNABLE, hasLock=false; OpenRegionProcedure 31ba9c474ceabbeaab02529db668ff6e, server=1cb9cf437a36,40661,1732777007755 because future has completed 2024-11-28T06:56:52,119 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=11, resume processing ppid=10 2024-11-28T06:56:52,119 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=11, ppid=10, state=SUCCESS, hasLock=false; OpenRegionProcedure 31ba9c474ceabbeaab02529db668ff6e, server=1cb9cf437a36,40661,1732777007755 in 176 msec 2024-11-28T06:56:52,122 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=10, resume processing ppid=7 2024-11-28T06:56:52,122 INFO [PEWorker-1 {}] procedure.ServerCrashProcedure(291): removed crashed server 1cb9cf437a36,46291,1732777007811 after splitting done 2024-11-28T06:56:52,122 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=10, ppid=7, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1732777009287, region=31ba9c474ceabbeaab02529db668ff6e, ASSIGN in 453 msec 2024-11-28T06:56:52,124 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=7, state=SUCCESS, hasLock=false; ServerCrashProcedure 1cb9cf437a36,46291,1732777007811, splitWal=true, meta=true in 883 msec 2024-11-28T06:56:52,140 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster 2024-11-28T06:56:52,140 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-28T06:56:52,140 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79) at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611) at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.io.hfile.TestBlockEvictionOnRegionMovement.tearDown(TestBlockEvictionOnRegionMovement.java:174) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-28T06:56:52,140 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T06:56:52,140 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T06:56:52,140 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-28T06:56:52,141 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2024-11-28T06:56:52,141 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=212569710, stopped=false 2024-11-28T06:56:52,141 INFO [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=1cb9cf437a36,42945,1732777007591 2024-11-28T06:56:52,191 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42945-0x10181846c940000, quorum=127.0.0.1:61052, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-28T06:56:52,191 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40661-0x10181846c940001, quorum=127.0.0.1:61052, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-28T06:56:52,191 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39675-0x10181846c940003, quorum=127.0.0.1:61052, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-28T06:56:52,192 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42945-0x10181846c940000, quorum=127.0.0.1:61052, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-28T06:56:52,192 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39675-0x10181846c940003, quorum=127.0.0.1:61052, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-28T06:56:52,192 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40661-0x10181846c940001, quorum=127.0.0.1:61052, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-28T06:56:52,192 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-28T06:56:52,192 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-28T06:56:52,192 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277) at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265) at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416) at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.io.hfile.TestBlockEvictionOnRegionMovement.tearDown(TestBlockEvictionOnRegionMovement.java:174) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-28T06:56:52,193 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T06:56:52,193 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:39675-0x10181846c940003, quorum=127.0.0.1:61052, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-28T06:56:52,193 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:42945-0x10181846c940000, quorum=127.0.0.1:61052, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-28T06:56:52,193 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '1cb9cf437a36,40661,1732777007755' ***** 2024-11-28T06:56:52,194 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-11-28T06:56:52,194 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '1cb9cf437a36,39675,1732777011493' ***** 2024-11-28T06:56:52,194 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-11-28T06:56:52,194 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:40661-0x10181846c940001, quorum=127.0.0.1:61052, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-28T06:56:52,194 INFO [RS:0;1cb9cf437a36:40661 {}] bucket.BucketCache(1738): Disabling cache 2024-11-28T06:56:52,194 INFO [RS:2;1cb9cf437a36:39675 {}] bucket.BucketCache(1738): Disabling cache 2024-11-28T06:56:52,195 INFO [Time-limited test-BucketCacheWriter-0 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-0 exiting, cacheEnabled=false 2024-11-28T06:56:52,195 INFO [RS:0;1cb9cf437a36:40661 {}] bucket.BucketCache(1762): Shutdown bucket cache: IO persistent=true; path to write=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/33d1270a-7ecb-879e-376a-4fd885e9a050/bucket.persistence 2024-11-28T06:56:52,195 INFO [RS:2;1cb9cf437a36:39675 {}] bucket.BucketCache(1762): Shutdown bucket cache: IO persistent=true; path to write=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/33d1270a-7ecb-879e-376a-4fd885e9a050/bucket.persistence 2024-11-28T06:56:52,195 INFO [Time-limited test-BucketCacheWriter-1 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-1 exiting, cacheEnabled=false 2024-11-28T06:56:52,195 INFO [Time-limited test-BucketCacheWriter-2 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-2 exiting, cacheEnabled=false 2024-11-28T06:56:52,195 INFO [Time-limited test-BucketCacheWriter-2 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-2 exiting, cacheEnabled=false 2024-11-28T06:56:52,195 INFO [RS:0;1cb9cf437a36:40661 {}] bucket.BucketCache(1768): Shutting down cache persister thread. 2024-11-28T06:56:52,195 INFO [Time-limited test-BucketCacheWriter-0 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-0 exiting, cacheEnabled=false 2024-11-28T06:56:52,196 INFO [Time-limited test-BucketCacheWriter-1 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-1 exiting, cacheEnabled=false 2024-11-28T06:56:52,195 WARN [bucket-cache-persister {}] bucket.BucketCachePersister(64): Interrupting BucketCachePersister thread. java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCachePersister.run(BucketCachePersister.java:45) ~[classes/:?] 2024-11-28T06:56:52,196 INFO [RS:2;1cb9cf437a36:39675 {}] bucket.BucketCache(1768): Shutting down cache persister thread. 2024-11-28T06:56:52,197 WARN [bucket-cache-persister {}] bucket.BucketCachePersister(64): Interrupting BucketCachePersister thread. java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCachePersister.run(BucketCachePersister.java:45) ~[classes/:?] 2024-11-28T06:56:52,204 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-28T06:56:52,204 INFO [regionserver/1cb9cf437a36:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-28T06:56:52,205 DEBUG [RS:0;1cb9cf437a36:40661 {}] bucket.BucketCache(1487): Thread RS:0;1cb9cf437a36:40661 started persisting bucket cache to file 2024-11-28T06:56:52,205 DEBUG [RS:0;1cb9cf437a36:40661 {}] bucket.BucketCache(1494): Persist in new chunked persistence format. 2024-11-28T06:56:52,206 DEBUG [RS:0;1cb9cf437a36:40661 {}] bucket.BucketCache(1673): persistToFile: before persisting backing map size: 0, fullycachedFiles size: 0, chunkSize: 10000 2024-11-28T06:56:52,206 DEBUG [RS:2;1cb9cf437a36:39675 {}] bucket.BucketCache(1487): Thread RS:2;1cb9cf437a36:39675 started persisting bucket cache to file 2024-11-28T06:56:52,207 DEBUG [RS:2;1cb9cf437a36:39675 {}] bucket.BucketCache(1494): Persist in new chunked persistence format. 2024-11-28T06:56:52,207 DEBUG [RS:2;1cb9cf437a36:39675 {}] bucket.BucketCache(1673): persistToFile: before persisting backing map size: 1, fullycachedFiles size: 1, chunkSize: 10000 2024-11-28T06:56:52,208 DEBUG [RS:0;1cb9cf437a36:40661 {}] bucket.BucketCache(1680): persistToFile: after persisting backing map size: 0, fullycachedFiles size: 0 2024-11-28T06:56:52,208 DEBUG [RS:0;1cb9cf437a36:40661 {}] bucket.BucketCache(1498): PersistToFile: after persisting backing map size: 0, fullycachedFiles size: 0, file name: bucket.persistence1732777012205 2024-11-28T06:56:52,209 DEBUG [RS:0;1cb9cf437a36:40661 {}] bucket.BucketCache(1509): Thread RS:0;1cb9cf437a36:40661 finished persisting bucket cache to file, renaming 2024-11-28T06:56:52,209 INFO [RS:0;1cb9cf437a36:40661 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-28T06:56:52,209 INFO [RS:0;1cb9cf437a36:40661 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-28T06:56:52,209 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-28T06:56:52,209 INFO [RS:0;1cb9cf437a36:40661 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-28T06:56:52,209 INFO [RS:0;1cb9cf437a36:40661 {}] regionserver.HRegionServer(3091): Received CLOSE for 31ba9c474ceabbeaab02529db668ff6e 2024-11-28T06:56:52,209 INFO [RS:0;1cb9cf437a36:40661 {}] regionserver.HRegionServer(959): stopping server 1cb9cf437a36,40661,1732777007755 2024-11-28T06:56:52,209 DEBUG [RS:2;1cb9cf437a36:39675 {}] bucket.BucketCache(1680): persistToFile: after persisting backing map size: 1, fullycachedFiles size: 1 2024-11-28T06:56:52,209 INFO [RS:0;1cb9cf437a36:40661 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-28T06:56:52,209 DEBUG [RS:2;1cb9cf437a36:39675 {}] bucket.BucketCache(1498): PersistToFile: after persisting backing map size: 1, fullycachedFiles size: 1, file name: bucket.persistence1732777012207 2024-11-28T06:56:52,209 INFO [RS:0;1cb9cf437a36:40661 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;1cb9cf437a36:40661. 2024-11-28T06:56:52,209 DEBUG [RS:2;1cb9cf437a36:39675 {}] bucket.BucketCache(1509): Thread RS:2;1cb9cf437a36:39675 finished persisting bucket cache to file, renaming 2024-11-28T06:56:52,210 DEBUG [RS:0;1cb9cf437a36:40661 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-28T06:56:52,210 INFO [RS_CLOSE_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1662): Closing region testBlockEvictionOnGracefulStop1732777009287,,1732777009293.31ba9c474ceabbeaab02529db668ff6e. during a graceful stop, and cache persistence is on, so setting evict on close to false. 2024-11-28T06:56:52,210 DEBUG [RS:0;1cb9cf437a36:40661 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T06:56:52,210 DEBUG [RS_CLOSE_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 31ba9c474ceabbeaab02529db668ff6e, disabling compactions & flushes 2024-11-28T06:56:52,210 INFO [RS:2;1cb9cf437a36:39675 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-28T06:56:52,210 INFO [RS_CLOSE_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testBlockEvictionOnGracefulStop1732777009287,,1732777009293.31ba9c474ceabbeaab02529db668ff6e. 2024-11-28T06:56:52,210 INFO [RS:0;1cb9cf437a36:40661 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-28T06:56:52,210 INFO [RS:2;1cb9cf437a36:39675 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-28T06:56:52,210 INFO [RS:0;1cb9cf437a36:40661 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-28T06:56:52,210 DEBUG [RS_CLOSE_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testBlockEvictionOnGracefulStop1732777009287,,1732777009293.31ba9c474ceabbeaab02529db668ff6e. 2024-11-28T06:56:52,210 INFO [RS:2;1cb9cf437a36:39675 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-28T06:56:52,210 INFO [RS:0;1cb9cf437a36:40661 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-28T06:56:52,210 INFO [RS:2;1cb9cf437a36:39675 {}] regionserver.HRegionServer(959): stopping server 1cb9cf437a36,39675,1732777011493 2024-11-28T06:56:52,210 DEBUG [RS_CLOSE_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testBlockEvictionOnGracefulStop1732777009287,,1732777009293.31ba9c474ceabbeaab02529db668ff6e. after waiting 0 ms 2024-11-28T06:56:52,210 INFO [RS:0;1cb9cf437a36:40661 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-11-28T06:56:52,210 INFO [RS:2;1cb9cf437a36:39675 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-28T06:56:52,210 DEBUG [RS_CLOSE_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testBlockEvictionOnGracefulStop1732777009287,,1732777009293.31ba9c474ceabbeaab02529db668ff6e. 2024-11-28T06:56:52,210 INFO [RS:2;1cb9cf437a36:39675 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:2;1cb9cf437a36:39675. 2024-11-28T06:56:52,210 INFO [RS:0;1cb9cf437a36:40661 {}] regionserver.HRegionServer(1321): Waiting on 2 regions to close 2024-11-28T06:56:52,210 DEBUG [RS:2;1cb9cf437a36:39675 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-28T06:56:52,210 DEBUG [RS:0;1cb9cf437a36:40661 {}] regionserver.HRegionServer(1325): Online Regions={1588230740=hbase:meta,,1.1588230740, 31ba9c474ceabbeaab02529db668ff6e=testBlockEvictionOnGracefulStop1732777009287,,1732777009293.31ba9c474ceabbeaab02529db668ff6e.} 2024-11-28T06:56:52,210 DEBUG [RS:2;1cb9cf437a36:39675 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T06:56:52,210 DEBUG [RS:0;1cb9cf437a36:40661 {}] regionserver.HRegionServer(1351): Waiting on 1588230740, 31ba9c474ceabbeaab02529db668ff6e 2024-11-28T06:56:52,210 INFO [RS:2;1cb9cf437a36:39675 {}] regionserver.HRegionServer(976): stopping server 1cb9cf437a36,39675,1732777011493; all regions closed. 2024-11-28T06:56:52,210 INFO [RS_CLOSE_META-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1662): Closing region hbase:meta,,1.1588230740 during a graceful stop, and cache persistence is on, so setting evict on close to false. 2024-11-28T06:56:52,210 DEBUG [RS_CLOSE_META-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-28T06:56:52,210 INFO [RS_CLOSE_META-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-28T06:56:52,211 DEBUG [RS_CLOSE_META-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-28T06:56:52,211 DEBUG [RS_CLOSE_META-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-28T06:56:52,211 DEBUG [RS_CLOSE_META-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-28T06:56:52,211 DEBUG [StoreFileCloser-31ba9c474ceabbeaab02529db668ff6e-family-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7e9a7384-d065-9ff8-196a-59688d3a4920/752619e7-664e-6a55-3bd6-93cba9c9ba92/data/default/testBlockEvictionOnGracefulStop1732777009287/31ba9c474ceabbeaab02529db668ff6e/family/68fcf27268094562965c0951b6ee0334 2024-11-28T06:56:52,211 INFO [RS_CLOSE_META-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=1.21 KB heapSize=2.80 KB 2024-11-28T06:56:52,212 DEBUG [RS:2;1cb9cf437a36:39675 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7e9a7384-d065-9ff8-196a-59688d3a4920/752619e7-664e-6a55-3bd6-93cba9c9ba92/oldWALs 2024-11-28T06:56:52,212 INFO [RS:2;1cb9cf437a36:39675 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 1cb9cf437a36%2C39675%2C1732777011493:(num 1732777011762) 2024-11-28T06:56:52,212 DEBUG [RS:2;1cb9cf437a36:39675 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T06:56:52,212 INFO [RS:2;1cb9cf437a36:39675 {}] regionserver.LeaseManager(133): Closed leases 2024-11-28T06:56:52,212 INFO [RS:2;1cb9cf437a36:39675 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-28T06:56:52,213 INFO [RS:2;1cb9cf437a36:39675 {}] hbase.ChoreService(370): Chore service for: regionserver/1cb9cf437a36:0 had [ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS] on shutdown 2024-11-28T06:56:52,213 INFO [RS:2;1cb9cf437a36:39675 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-28T06:56:52,213 INFO [RS:2;1cb9cf437a36:39675 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-28T06:56:52,213 INFO [RS:2;1cb9cf437a36:39675 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-28T06:56:52,213 INFO [RS:2;1cb9cf437a36:39675 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-28T06:56:52,213 INFO [RS:2;1cb9cf437a36:39675 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:39675 2024-11-28T06:56:52,214 INFO [regionserver/1cb9cf437a36:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-28T06:56:52,215 DEBUG [RS_CLOSE_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7e9a7384-d065-9ff8-196a-59688d3a4920/752619e7-664e-6a55-3bd6-93cba9c9ba92/data/default/testBlockEvictionOnGracefulStop1732777009287/31ba9c474ceabbeaab02529db668ff6e/recovered.edits/12.seqid, newMaxSeqId=12, maxSeqId=9 2024-11-28T06:56:52,215 INFO [RS_CLOSE_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testBlockEvictionOnGracefulStop1732777009287,,1732777009293.31ba9c474ceabbeaab02529db668ff6e. 2024-11-28T06:56:52,215 DEBUG [RS_CLOSE_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 31ba9c474ceabbeaab02529db668ff6e: Waiting for close lock at 1732777012209Running coprocessor pre-close hooks at 1732777012210 (+1 ms)Disabling compacts and flushes for region at 1732777012210Disabling writes for close at 1732777012210Writing region close event to WAL at 1732777012211 (+1 ms)Running coprocessor post-close hooks at 1732777012215 (+4 ms)Closed at 1732777012215 2024-11-28T06:56:52,215 DEBUG [RS_CLOSE_REGION-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testBlockEvictionOnGracefulStop1732777009287,,1732777009293.31ba9c474ceabbeaab02529db668ff6e. 2024-11-28T06:56:52,218 INFO [regionserver/1cb9cf437a36:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-28T06:56:52,221 DEBUG [RS_CLOSE_META-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 98072c84bd874ec8af29c00f74333ae8 2024-11-28T06:56:52,231 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42945-0x10181846c940000, quorum=127.0.0.1:61052, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-28T06:56:52,231 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39675-0x10181846c940003, quorum=127.0.0.1:61052, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/1cb9cf437a36,39675,1732777011493 2024-11-28T06:56:52,231 INFO [RS:2;1cb9cf437a36:39675 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-28T06:56:52,234 DEBUG [RS_CLOSE_META-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7e9a7384-d065-9ff8-196a-59688d3a4920/752619e7-664e-6a55-3bd6-93cba9c9ba92/data/hbase/meta/1588230740/.tmp/info/98072c84bd874ec8af29c00f74333ae8 is 209, key is testBlockEvictionOnGracefulStop1732777009287,,1732777009293.31ba9c474ceabbeaab02529db668ff6e./info:regioninfo/1732777012113/Put/seqid=0 2024-11-28T06:56:52,234 INFO [RS_CLOSE_META-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.21 KB at sequenceid=19 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7e9a7384-d065-9ff8-196a-59688d3a4920/752619e7-664e-6a55-3bd6-93cba9c9ba92/data/hbase/meta/1588230740/.tmp/info/98072c84bd874ec8af29c00f74333ae8 2024-11-28T06:56:52,236 DEBUG [RS_CLOSE_META-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7e9a7384-d065-9ff8-196a-59688d3a4920/752619e7-664e-6a55-3bd6-93cba9c9ba92/data/hbase/meta/1588230740/.tmp/info/98072c84bd874ec8af29c00f74333ae8 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7e9a7384-d065-9ff8-196a-59688d3a4920/752619e7-664e-6a55-3bd6-93cba9c9ba92/data/hbase/meta/1588230740/info/98072c84bd874ec8af29c00f74333ae8 2024-11-28T06:56:52,237 DEBUG [RS_CLOSE_META-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_CLOSE_META}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7e9a7384-d065-9ff8-196a-59688d3a4920/752619e7-664e-6a55-3bd6-93cba9c9ba92/data/hbase/meta/1588230740/info/98072c84bd874ec8af29c00f74333ae8, delay=954 ms 2024-11-28T06:56:52,238 INFO [RS_CLOSE_META-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7e9a7384-d065-9ff8-196a-59688d3a4920/752619e7-664e-6a55-3bd6-93cba9c9ba92/data/hbase/meta/1588230740/info/98072c84bd874ec8af29c00f74333ae8, entries=8, sequenceid=19, filesize=6.6 K 2024-11-28T06:56:52,239 INFO [RS_CLOSE_META-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~1.21 KB/1238, heapSize ~2.03 KB/2080, currentSize=0 B/0 for 1588230740 in 28ms, sequenceid=19, compaction requested=false 2024-11-28T06:56:52,240 DEBUG [StoreFileCloser-1588230740-info-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7e9a7384-d065-9ff8-196a-59688d3a4920/752619e7-664e-6a55-3bd6-93cba9c9ba92/data/hbase/meta/1588230740/info/666397d4cded4a028775335b64d1e84e 2024-11-28T06:56:52,240 DEBUG [StoreFileCloser-1588230740-info-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7e9a7384-d065-9ff8-196a-59688d3a4920/752619e7-664e-6a55-3bd6-93cba9c9ba92/data/hbase/meta/1588230740/info/98072c84bd874ec8af29c00f74333ae8 2024-11-28T06:56:52,240 DEBUG [StoreFileCloser-1588230740-ns-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7e9a7384-d065-9ff8-196a-59688d3a4920/752619e7-664e-6a55-3bd6-93cba9c9ba92/data/hbase/meta/1588230740/ns/4ef821c80f9a4e05ab163ae8a76012e3 2024-11-28T06:56:52,241 DEBUG [StoreFileCloser-1588230740-table-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7e9a7384-d065-9ff8-196a-59688d3a4920/752619e7-664e-6a55-3bd6-93cba9c9ba92/data/hbase/meta/1588230740/table/9b2097e829dc4962a8584e9c1d2359d8 2024-11-28T06:56:52,245 DEBUG [RS_CLOSE_META-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7e9a7384-d065-9ff8-196a-59688d3a4920/752619e7-664e-6a55-3bd6-93cba9c9ba92/data/hbase/meta/1588230740/recovered.edits/22.seqid, newMaxSeqId=22, maxSeqId=14 2024-11-28T06:56:52,245 DEBUG [RS_CLOSE_META-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-28T06:56:52,245 INFO [RS_CLOSE_META-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-28T06:56:52,245 DEBUG [RS_CLOSE_META-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1732777012210Running coprocessor pre-close hooks at 1732777012210Disabling compacts and flushes for region at 1732777012210Disabling writes for close at 1732777012211 (+1 ms)Obtaining lock to block concurrent updates at 1732777012211Preparing flush snapshotting stores in 1588230740 at 1732777012211Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=1238, getHeapSize=2800, getOffHeapSize=0, getCellsCount=8 at 1732777012211Flushing stores of hbase:meta,,1.1588230740 at 1732777012212 (+1 ms)Flushing 1588230740/info: creating writer at 1732777012212Flushing 1588230740/info: appending metadata at 1732777012233 (+21 ms)Flushing 1588230740/info: closing flushed file at 1732777012233Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@558db137: reopening flushed file at 1732777012236 (+3 ms)Finished flush of dataSize ~1.21 KB/1238, heapSize ~2.03 KB/2080, currentSize=0 B/0 for 1588230740 in 28ms, sequenceid=19, compaction requested=false at 1732777012239 (+3 ms)Writing region close event to WAL at 1732777012241 (+2 ms)Running coprocessor post-close hooks at 1732777012245 (+4 ms)Closed at 1732777012245 2024-11-28T06:56:52,245 DEBUG [RS_CLOSE_META-regionserver/1cb9cf437a36:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-11-28T06:56:52,245 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [1cb9cf437a36,39675,1732777011493] 2024-11-28T06:56:52,253 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/1cb9cf437a36,39675,1732777011493 already deleted, retry=false 2024-11-28T06:56:52,254 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; 1cb9cf437a36,39675,1732777011493 expired; onlineServers=1 2024-11-28T06:56:52,346 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39675-0x10181846c940003, quorum=127.0.0.1:61052, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-28T06:56:52,346 INFO [RS:2;1cb9cf437a36:39675 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-28T06:56:52,346 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39675-0x10181846c940003, quorum=127.0.0.1:61052, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-28T06:56:52,346 INFO [RS:2;1cb9cf437a36:39675 {}] regionserver.HRegionServer(1031): Exiting; stopping=1cb9cf437a36,39675,1732777011493; zookeeper connection closed. 2024-11-28T06:56:52,346 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@4a336f88 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@4a336f88 2024-11-28T06:56:52,410 INFO [RS:0;1cb9cf437a36:40661 {}] regionserver.HRegionServer(976): stopping server 1cb9cf437a36,40661,1732777007755; all regions closed. 2024-11-28T06:56:52,413 DEBUG [RS:0;1cb9cf437a36:40661 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7e9a7384-d065-9ff8-196a-59688d3a4920/752619e7-664e-6a55-3bd6-93cba9c9ba92/oldWALs 2024-11-28T06:56:52,413 INFO [RS:0;1cb9cf437a36:40661 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 1cb9cf437a36%2C40661%2C1732777007755.meta:.meta(num 1732777011596) 2024-11-28T06:56:52,414 DEBUG [RS:0;1cb9cf437a36:40661 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7e9a7384-d065-9ff8-196a-59688d3a4920/752619e7-664e-6a55-3bd6-93cba9c9ba92/oldWALs 2024-11-28T06:56:52,415 INFO [RS:0;1cb9cf437a36:40661 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 1cb9cf437a36%2C40661%2C1732777007755:(num 1732777008663) 2024-11-28T06:56:52,415 DEBUG [RS:0;1cb9cf437a36:40661 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T06:56:52,415 INFO [RS:0;1cb9cf437a36:40661 {}] regionserver.LeaseManager(133): Closed leases 2024-11-28T06:56:52,415 INFO [RS:0;1cb9cf437a36:40661 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-28T06:56:52,415 INFO [RS:0;1cb9cf437a36:40661 {}] hbase.ChoreService(370): Chore service for: regionserver/1cb9cf437a36:0 had [ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS] on shutdown 2024-11-28T06:56:52,415 INFO [RS:0;1cb9cf437a36:40661 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-28T06:56:52,415 INFO [regionserver/1cb9cf437a36:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-28T06:56:52,416 INFO [RS:0;1cb9cf437a36:40661 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:40661 2024-11-28T06:56:52,441 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40661-0x10181846c940001, quorum=127.0.0.1:61052, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/1cb9cf437a36,40661,1732777007755 2024-11-28T06:56:52,441 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42945-0x10181846c940000, quorum=127.0.0.1:61052, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-28T06:56:52,441 INFO [RS:0;1cb9cf437a36:40661 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-28T06:56:52,449 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [1cb9cf437a36,40661,1732777007755] 2024-11-28T06:56:52,458 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/1cb9cf437a36,40661,1732777007755 already deleted, retry=false 2024-11-28T06:56:52,458 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; 1cb9cf437a36,40661,1732777007755 expired; onlineServers=0 2024-11-28T06:56:52,458 INFO [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master '1cb9cf437a36,42945,1732777007591' ***** 2024-11-28T06:56:52,458 INFO [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0 2024-11-28T06:56:52,458 INFO [M:0;1cb9cf437a36:42945 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-28T06:56:52,458 INFO [M:0;1cb9cf437a36:42945 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-28T06:56:52,458 DEBUG [M:0;1cb9cf437a36:42945 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2024-11-28T06:56:52,458 DEBUG [M:0;1cb9cf437a36:42945 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2024-11-28T06:56:52,458 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2024-11-28T06:56:52,458 DEBUG [master/1cb9cf437a36:0:becomeActiveMaster-HFileCleaner.small.0-1732777008375 {}] cleaner.HFileCleaner(306): Exit Thread[master/1cb9cf437a36:0:becomeActiveMaster-HFileCleaner.small.0-1732777008375,5,FailOnTimeoutGroup] 2024-11-28T06:56:52,458 DEBUG [master/1cb9cf437a36:0:becomeActiveMaster-HFileCleaner.large.0-1732777008375 {}] cleaner.HFileCleaner(306): Exit Thread[master/1cb9cf437a36:0:becomeActiveMaster-HFileCleaner.large.0-1732777008375,5,FailOnTimeoutGroup] 2024-11-28T06:56:52,458 INFO [M:0;1cb9cf437a36:42945 {}] hbase.ChoreService(370): Chore service for: master/1cb9cf437a36:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown 2024-11-28T06:56:52,459 INFO [M:0;1cb9cf437a36:42945 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-28T06:56:52,459 DEBUG [M:0;1cb9cf437a36:42945 {}] master.HMaster(1795): Stopping service threads 2024-11-28T06:56:52,459 INFO [M:0;1cb9cf437a36:42945 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2024-11-28T06:56:52,459 INFO [M:0;1cb9cf437a36:42945 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-28T06:56:52,459 INFO [M:0;1cb9cf437a36:42945 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-28T06:56:52,459 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2024-11-28T06:56:52,466 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42945-0x10181846c940000, quorum=127.0.0.1:61052, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-11-28T06:56:52,466 DEBUG [M:0;1cb9cf437a36:42945 {}] zookeeper.ZKUtil(347): master:42945-0x10181846c940000, quorum=127.0.0.1:61052, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error) 2024-11-28T06:56:52,466 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42945-0x10181846c940000, quorum=127.0.0.1:61052, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-28T06:56:52,466 WARN [M:0;1cb9cf437a36:42945 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null 2024-11-28T06:56:52,466 INFO [M:0;1cb9cf437a36:42945 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7e9a7384-d065-9ff8-196a-59688d3a4920/752619e7-664e-6a55-3bd6-93cba9c9ba92/.lastflushedseqids 2024-11-28T06:56:52,472 INFO [M:0;1cb9cf437a36:42945 {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-28T06:56:52,472 INFO [M:0;1cb9cf437a36:42945 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2024-11-28T06:56:52,472 DEBUG [M:0;1cb9cf437a36:42945 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-28T06:56:52,472 INFO [M:0;1cb9cf437a36:42945 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-28T06:56:52,472 DEBUG [M:0;1cb9cf437a36:42945 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-28T06:56:52,472 DEBUG [M:0;1cb9cf437a36:42945 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-28T06:56:52,472 DEBUG [M:0;1cb9cf437a36:42945 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-28T06:56:52,472 INFO [M:0;1cb9cf437a36:42945 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=39.49 KB heapSize=49.88 KB 2024-11-28T06:56:52,480 DEBUG [M:0;1cb9cf437a36:42945 {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on dce3dd150b0f4c8894a34be2236f5885 2024-11-28T06:56:52,492 DEBUG [M:0;1cb9cf437a36:42945 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7e9a7384-d065-9ff8-196a-59688d3a4920/752619e7-664e-6a55-3bd6-93cba9c9ba92/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/dce3dd150b0f4c8894a34be2236f5885 is 82, key is hbase:meta,,1/info:regioninfo/1732777011637/Put/seqid=0 2024-11-28T06:56:52,493 INFO [M:0;1cb9cf437a36:42945 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1008 B at sequenceid=102 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7e9a7384-d065-9ff8-196a-59688d3a4920/752619e7-664e-6a55-3bd6-93cba9c9ba92/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/dce3dd150b0f4c8894a34be2236f5885 2024-11-28T06:56:52,499 DEBUG [M:0;1cb9cf437a36:42945 {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 053a55a86e0a4952b7a336e5b3fa4eb3 2024-11-28T06:56:52,512 DEBUG [M:0;1cb9cf437a36:42945 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7e9a7384-d065-9ff8-196a-59688d3a4920/752619e7-664e-6a55-3bd6-93cba9c9ba92/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/053a55a86e0a4952b7a336e5b3fa4eb3 is 781, key is \x00\x00\x00\x00\x00\x00\x00\x04/proc:d/1732777009687/Put/seqid=0 2024-11-28T06:56:52,512 INFO [M:0;1cb9cf437a36:42945 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.26 KB at sequenceid=102 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7e9a7384-d065-9ff8-196a-59688d3a4920/752619e7-664e-6a55-3bd6-93cba9c9ba92/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/053a55a86e0a4952b7a336e5b3fa4eb3 2024-11-28T06:56:52,519 DEBUG [M:0;1cb9cf437a36:42945 {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 7482701e5c8b43638ddd92c22ee6c9e3 2024-11-28T06:56:52,532 DEBUG [M:0;1cb9cf437a36:42945 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7e9a7384-d065-9ff8-196a-59688d3a4920/752619e7-664e-6a55-3bd6-93cba9c9ba92/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/7482701e5c8b43638ddd92c22ee6c9e3 is 69, key is 1cb9cf437a36,39675,1732777011493/rs:state/1732777011577/Put/seqid=0 2024-11-28T06:56:52,533 INFO [M:0;1cb9cf437a36:42945 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=249 B at sequenceid=102 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7e9a7384-d065-9ff8-196a-59688d3a4920/752619e7-664e-6a55-3bd6-93cba9c9ba92/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/7482701e5c8b43638ddd92c22ee6c9e3 2024-11-28T06:56:52,535 INFO [M:0;1cb9cf437a36:42945 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 7482701e5c8b43638ddd92c22ee6c9e3 2024-11-28T06:56:52,535 DEBUG [M:0;1cb9cf437a36:42945 {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7e9a7384-d065-9ff8-196a-59688d3a4920/752619e7-664e-6a55-3bd6-93cba9c9ba92/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/dce3dd150b0f4c8894a34be2236f5885 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7e9a7384-d065-9ff8-196a-59688d3a4920/752619e7-664e-6a55-3bd6-93cba9c9ba92/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/dce3dd150b0f4c8894a34be2236f5885 2024-11-28T06:56:52,537 INFO [M:0;1cb9cf437a36:42945 {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7e9a7384-d065-9ff8-196a-59688d3a4920/752619e7-664e-6a55-3bd6-93cba9c9ba92/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/dce3dd150b0f4c8894a34be2236f5885, entries=14, sequenceid=102, filesize=5.9 K 2024-11-28T06:56:52,537 DEBUG [M:0;1cb9cf437a36:42945 {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7e9a7384-d065-9ff8-196a-59688d3a4920/752619e7-664e-6a55-3bd6-93cba9c9ba92/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/053a55a86e0a4952b7a336e5b3fa4eb3 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7e9a7384-d065-9ff8-196a-59688d3a4920/752619e7-664e-6a55-3bd6-93cba9c9ba92/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/053a55a86e0a4952b7a336e5b3fa4eb3 2024-11-28T06:56:52,538 INFO [M:0;1cb9cf437a36:42945 {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7e9a7384-d065-9ff8-196a-59688d3a4920/752619e7-664e-6a55-3bd6-93cba9c9ba92/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/053a55a86e0a4952b7a336e5b3fa4eb3, entries=11, sequenceid=102, filesize=6.5 K 2024-11-28T06:56:52,538 DEBUG [M:0;1cb9cf437a36:42945 {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7e9a7384-d065-9ff8-196a-59688d3a4920/752619e7-664e-6a55-3bd6-93cba9c9ba92/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/7482701e5c8b43638ddd92c22ee6c9e3 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7e9a7384-d065-9ff8-196a-59688d3a4920/752619e7-664e-6a55-3bd6-93cba9c9ba92/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/7482701e5c8b43638ddd92c22ee6c9e3 2024-11-28T06:56:52,540 INFO [M:0;1cb9cf437a36:42945 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 7482701e5c8b43638ddd92c22ee6c9e3 2024-11-28T06:56:52,540 INFO [M:0;1cb9cf437a36:42945 {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7e9a7384-d065-9ff8-196a-59688d3a4920/752619e7-664e-6a55-3bd6-93cba9c9ba92/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/7482701e5c8b43638ddd92c22ee6c9e3, entries=3, sequenceid=102, filesize=5.3 K 2024-11-28T06:56:52,541 INFO [M:0;1cb9cf437a36:42945 {}] regionserver.HRegion(3140): Finished flush of dataSize ~39.49 KB/40436, heapSize ~49.59 KB/50776, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 68ms, sequenceid=102, compaction requested=false 2024-11-28T06:56:52,542 INFO [M:0;1cb9cf437a36:42945 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-28T06:56:52,543 DEBUG [M:0;1cb9cf437a36:42945 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1732777012472Disabling compacts and flushes for region at 1732777012472Disabling writes for close at 1732777012472Obtaining lock to block concurrent updates at 1732777012472Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1732777012472Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=40436, getHeapSize=51016, getOffHeapSize=0, getCellsCount=127 at 1732777012473 (+1 ms)Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1732777012473Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1732777012473Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1732777012492 (+19 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1732777012492Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1732777012494 (+2 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1732777012511 (+17 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1732777012511Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1732777012514 (+3 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1732777012532 (+18 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1732777012532Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@3a4f2af4: reopening flushed file at 1732777012535 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@19f90e1f: reopening flushed file at 1732777012537 (+2 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@ab408f6: reopening flushed file at 1732777012538 (+1 ms)Finished flush of dataSize ~39.49 KB/40436, heapSize ~49.59 KB/50776, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 68ms, sequenceid=102, compaction requested=false at 1732777012541 (+3 ms)Writing region close event to WAL at 1732777012542 (+1 ms)Closed at 1732777012542 2024-11-28T06:56:52,544 INFO [M:0;1cb9cf437a36:42945 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down. 2024-11-28T06:56:52,544 INFO [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-28T06:56:52,544 INFO [M:0;1cb9cf437a36:42945 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:42945 2024-11-28T06:56:52,544 INFO [M:0;1cb9cf437a36:42945 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-28T06:56:52,550 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40661-0x10181846c940001, quorum=127.0.0.1:61052, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-28T06:56:52,550 INFO [RS:0;1cb9cf437a36:40661 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-28T06:56:52,550 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40661-0x10181846c940001, quorum=127.0.0.1:61052, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-28T06:56:52,550 INFO [RS:0;1cb9cf437a36:40661 {}] regionserver.HRegionServer(1031): Exiting; stopping=1cb9cf437a36,40661,1732777007755; zookeeper connection closed. 2024-11-28T06:56:52,550 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@23dadd72 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@23dadd72 2024-11-28T06:56:52,550 INFO [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 3 regionserver(s) complete 2024-11-28T06:56:52,658 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42945-0x10181846c940000, quorum=127.0.0.1:61052, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-28T06:56:52,658 INFO [M:0;1cb9cf437a36:42945 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-28T06:56:52,659 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42945-0x10181846c940000, quorum=127.0.0.1:61052, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-28T06:56:52,666 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers 2024-11-28T06:56:52,668 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down 2024-11-28T06:56:52,668 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-28T06:56:52,675 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: io.hfile.TestBlockEvictionOnRegionMovement#testBlockEvictionOnGracefulStop Thread=72 (was 52) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-8-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-6 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HMaster-EventLoopGroup-6-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HMaster-EventLoopGroup-6-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-9-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-7-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-8-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-7-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-8 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-7 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-9 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-8-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-7-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HMaster-EventLoopGroup-6-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-9-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=382 (was 346) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=347 (was 368), ProcessCount=11 (was 11), AvailableMemoryMB=8032 (was 8129)