2024-11-08 19:22:15,332 main DEBUG Apache Log4j Core 2.17.2 initializing configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@5745ca0e 2024-11-08 19:22:15,353 main DEBUG Took 0.018644 seconds to load 1 plugins from package org.apache.hadoop.hbase.logging 2024-11-08 19:22:15,353 main DEBUG PluginManager 'Core' found 129 plugins 2024-11-08 19:22:15,354 main DEBUG PluginManager 'Level' found 0 plugins 2024-11-08 19:22:15,355 main DEBUG PluginManager 'Lookup' found 16 plugins 2024-11-08 19:22:15,356 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-08 19:22:15,364 main DEBUG PluginManager 'TypeConverter' found 26 plugins 2024-11-08 19:22:15,381 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.util.MBeans", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-08 19:22:15,382 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-08 19:22:15,383 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.logging.TestJul2Slf4j", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-08 19:22:15,384 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-08 19:22:15,384 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.zookeeper", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-08 19:22:15,385 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-08 19:22:15,386 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSinkAdapter", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-08 19:22:15,387 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-08 19:22:15,387 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSystemImpl", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-08 19:22:15,388 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-08 19:22:15,389 main DEBUG LoggerConfig$Builder(additivity="false", level="WARN", levelAndRefs="null", name="org.apache.directory", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-08 19:22:15,389 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-08 19:22:15,390 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.ipc.FailedServers", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-08 19:22:15,390 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-08 19:22:15,391 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsConfig", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-08 19:22:15,391 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-08 19:22:15,392 main DEBUG LoggerConfig$Builder(additivity="null", level="INFO", levelAndRefs="null", name="org.apache.hadoop.hbase.ScheduledChore", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-08 19:22:15,392 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-08 19:22:15,393 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.regionserver.RSRpcServices", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-08 19:22:15,393 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-08 19:22:15,394 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-08 19:22:15,394 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-08 19:22:15,394 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-08 19:22:15,395 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-08 19:22:15,395 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hbase.thirdparty.io.netty.channel", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-08 19:22:15,395 main DEBUG Building Plugin[name=root, class=org.apache.logging.log4j.core.config.LoggerConfig$RootLogger]. 2024-11-08 19:22:15,397 main DEBUG LoggerConfig$RootLogger$Builder(additivity="null", level="null", levelAndRefs="INFO,Console", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-08 19:22:15,398 main DEBUG Building Plugin[name=loggers, class=org.apache.logging.log4j.core.config.LoggersPlugin]. 2024-11-08 19:22:15,400 main DEBUG createLoggers(={org.apache.hadoop.metrics2.util.MBeans, org.apache.hadoop.hbase.logging.TestJul2Slf4j, org.apache.zookeeper, org.apache.hadoop.metrics2.impl.MetricsSinkAdapter, org.apache.hadoop.metrics2.impl.MetricsSystemImpl, org.apache.directory, org.apache.hadoop.hbase.ipc.FailedServers, org.apache.hadoop.metrics2.impl.MetricsConfig, org.apache.hadoop.hbase.ScheduledChore, org.apache.hadoop.hbase.regionserver.RSRpcServices, org.apache.hadoop, org.apache.hadoop.hbase, org.apache.hbase.thirdparty.io.netty.channel, root}) 2024-11-08 19:22:15,400 main DEBUG Building Plugin[name=layout, class=org.apache.logging.log4j.core.layout.PatternLayout]. 2024-11-08 19:22:15,402 main DEBUG PatternLayout$Builder(pattern="%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n", PatternSelector=null, Configuration(PropertiesConfig), Replace=null, charset="null", alwaysWriteExceptions="null", disableAnsi="null", noConsoleNoAnsi="null", header="null", footer="null") 2024-11-08 19:22:15,402 main DEBUG PluginManager 'Converter' found 47 plugins 2024-11-08 19:22:15,410 main DEBUG Building Plugin[name=appender, class=org.apache.hadoop.hbase.logging.HBaseTestAppender]. 2024-11-08 19:22:15,412 main DEBUG HBaseTestAppender$Builder(target="SYSTEM_ERR", maxSize="1G", bufferedIo="null", bufferSize="null", immediateFlush="null", ignoreExceptions="null", PatternLayout(%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n), name="Console", Configuration(PropertiesConfig), Filter=null, ={}) 2024-11-08 19:22:15,414 main DEBUG Starting HBaseTestOutputStreamManager SYSTEM_ERR 2024-11-08 19:22:15,414 main DEBUG Building Plugin[name=appenders, class=org.apache.logging.log4j.core.config.AppendersPlugin]. 2024-11-08 19:22:15,415 main DEBUG createAppenders(={Console}) 2024-11-08 19:22:15,415 main DEBUG Configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@5745ca0e initialized 2024-11-08 19:22:15,415 main DEBUG Starting configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@5745ca0e 2024-11-08 19:22:15,416 main DEBUG Started configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@5745ca0e OK. 2024-11-08 19:22:15,416 main DEBUG Shutting down OutputStreamManager SYSTEM_OUT.false.false-1 2024-11-08 19:22:15,416 main DEBUG OutputStream closed 2024-11-08 19:22:15,417 main DEBUG Shut down OutputStreamManager SYSTEM_OUT.false.false-1, all resources released: true 2024-11-08 19:22:15,417 main DEBUG Appender DefaultConsole-1 stopped with status true 2024-11-08 19:22:15,417 main DEBUG Stopped org.apache.logging.log4j.core.config.DefaultConfiguration@74751b3 OK 2024-11-08 19:22:15,500 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6 2024-11-08 19:22:15,502 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=StatusLogger 2024-11-08 19:22:15,504 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=ContextSelector 2024-11-08 19:22:15,505 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name= 2024-11-08 19:22:15,506 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.directory 2024-11-08 19:22:15,506 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSinkAdapter 2024-11-08 19:22:15,507 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.zookeeper 2024-11-08 19:22:15,507 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.logging.TestJul2Slf4j 2024-11-08 19:22:15,508 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSystemImpl 2024-11-08 19:22:15,508 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.util.MBeans 2024-11-08 19:22:15,509 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase 2024-11-08 19:22:15,509 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop 2024-11-08 19:22:15,509 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ipc.FailedServers 2024-11-08 19:22:15,510 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.regionserver.RSRpcServices 2024-11-08 19:22:15,510 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsConfig 2024-11-08 19:22:15,510 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hbase.thirdparty.io.netty.channel 2024-11-08 19:22:15,511 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ScheduledChore 2024-11-08 19:22:15,512 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Appenders,name=Console 2024-11-08 19:22:15,514 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-08 19:22:15,514 main DEBUG Reconfiguration complete for context[name=1dbd16a6] at URI jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-logging/target/hbase-logging-3.0.0-beta-2-SNAPSHOT-tests.jar!/log4j2.properties (org.apache.logging.log4j.core.LoggerContext@528c868) with optional ClassLoader: null 2024-11-08 19:22:15,514 main DEBUG Shutdown hook enabled. Registering a new one. 2024-11-08 19:22:15,515 main DEBUG LoggerContext[name=1dbd16a6, org.apache.logging.log4j.core.LoggerContext@528c868] started OK. 2024-11-08T19:22:15,528 INFO [main {}] hbase.HBaseClassTestRule(94): Test class org.apache.hadoop.hbase.io.hfile.TestPrefetchRSClose timeout: 13 mins 2024-11-08 19:22:15,530 main DEBUG AsyncLogger.ThreadNameStrategy=UNCACHED (user specified null, default is UNCACHED) 2024-11-08 19:22:15,530 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-08T19:22:15,761 DEBUG [main {}] hbase.HBaseTestingUtil(323): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/8a0ec66d-a62f-b9fa-51db-a1c5c34d17d3 2024-11-08T19:22:15,785 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: io.hfile.TestPrefetchRSClose#testPrefetchPersistence Thread=12, OpenFileDescriptor=287, MaxFileDescriptor=1048576, SystemLoadAverage=424, ProcessCount=11, AvailableMemoryMB=3271 2024-11-08T19:22:15,874 WARN [Time-limited test {}] util.NativeCodeLoader(60): Unable to load native-hadoop library for your platform... using builtin-java classes where applicable 2024-11-08T19:22:15,962 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-08T19:22:15,976 INFO [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/8a0ec66d-a62f-b9fa-51db-a1c5c34d17d3/cluster_b322d7bf-b2de-843b-597f-3cd84b9e6563, deleteOnExit=true 2024-11-08T19:22:16,107 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/8a0ec66d-a62f-b9fa-51db-a1c5c34d17d3/cluster_b322d7bf-b2de-843b-597f-3cd84b9e6563/zookeeper_0, clientPort=63355, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/8a0ec66d-a62f-b9fa-51db-a1c5c34d17d3/cluster_b322d7bf-b2de-843b-597f-3cd84b9e6563/zookeeper_0/version-2, dataDirSize=457 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/8a0ec66d-a62f-b9fa-51db-a1c5c34d17d3/cluster_b322d7bf-b2de-843b-597f-3cd84b9e6563/zookeeper_0/version-2, dataLogSize=457 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0 2024-11-08T19:22:16,121 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=63355 2024-11-08T19:22:16,122 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-08T19:22:16,231 INFO [Time-limited test {}] util.FSUtils(489): Created version file at /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/8a0ec66d-a62f-b9fa-51db-a1c5c34d17d3/f4a1b344-4599-9a20-81eb-9d7a968f2df2 with version=8 2024-11-08T19:22:16,232 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1139): Setting hbase.fs.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/8a0ec66d-a62f-b9fa-51db-a1c5c34d17d3/f4a1b344-4599-9a20-81eb-9d7a968f2df2/hbase-staging 2024-11-08T19:22:16,327 DEBUG [Time-limited test {}] channel.MultithreadEventLoopGroup(44): -Dio.netty.eventLoopThreads: 16 2024-11-08T19:22:16,550 INFO [Time-limited test {}] client.ConnectionUtils(128): master/692b295ad45c:0 server-side Connection retries=45 2024-11-08T19:22:16,559 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-08T19:22:16,560 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-08T19:22:16,567 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-08T19:22:16,567 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-08T19:22:16,567 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-08T19:22:16,705 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService 2024-11-08T19:22:16,793 WARN [Time-limited test {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-08T19:22:16,836 INFO [Time-limited test {}] metrics.MetricRegistriesLoader(60): Loaded MetricRegistries class org.apache.hadoop.hbase.metrics.impl.MetricRegistriesImpl 2024-11-08T19:22:16,847 DEBUG [Time-limited test {}] util.ClassSize(228): Using Unsafe to estimate memory layout 2024-11-08T19:22:16,850 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-08T19:22:16,891 DEBUG [Time-limited test {}] channel.DefaultChannelId(84): -Dio.netty.processId: 30357 (auto-detected) 2024-11-08T19:22:16,892 DEBUG [Time-limited test {}] channel.DefaultChannelId(106): -Dio.netty.machineId: 02:42:ac:ff:fe:11:00:02 (auto-detected) 2024-11-08T19:22:16,917 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:38529 2024-11-08T19:22:16,947 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:38529 connecting to ZooKeeper ensemble=127.0.0.1:63355 2024-11-08T19:22:17,026 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:385290x0, quorum=127.0.0.1:63355, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-08T19:22:17,031 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:38529-0x1011be626410000 connected 2024-11-08T19:22:17,132 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-08T19:22:17,134 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-08T19:22:17,148 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:38529-0x1011be626410000, quorum=127.0.0.1:63355, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-08T19:22:17,151 INFO [Time-limited test {}] master.HMaster(525): hbase.rootdir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/8a0ec66d-a62f-b9fa-51db-a1c5c34d17d3/f4a1b344-4599-9a20-81eb-9d7a968f2df2, hbase.cluster.distributed=false 2024-11-08T19:22:17,174 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:38529-0x1011be626410000, quorum=127.0.0.1:63355, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-08T19:22:17,179 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=38529 2024-11-08T19:22:17,180 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=38529 2024-11-08T19:22:17,180 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=38529 2024-11-08T19:22:17,183 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=38529 2024-11-08T19:22:17,183 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=38529 2024-11-08T19:22:17,259 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/692b295ad45c:0 server-side Connection retries=45 2024-11-08T19:22:17,260 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-08T19:22:17,261 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-08T19:22:17,261 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-08T19:22:17,261 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-08T19:22:17,261 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-08T19:22:17,264 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-08T19:22:17,266 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-08T19:22:17,267 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:37371 2024-11-08T19:22:17,269 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:37371 connecting to ZooKeeper ensemble=127.0.0.1:63355 2024-11-08T19:22:17,270 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-08T19:22:17,271 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-08T19:22:17,285 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:373710x0, quorum=127.0.0.1:63355, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-08T19:22:17,286 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:37371-0x1011be626410001 connected 2024-11-08T19:22:17,286 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:37371-0x1011be626410001, quorum=127.0.0.1:63355, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-08T19:22:17,293 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-08T19:22:17,306 INFO [Time-limited test {}] bucket.FileIOEngine(99): Allocating cache 400 MB, on the path:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/8a0ec66d-a62f-b9fa-51db-a1c5c34d17d3/bucket.cache 2024-11-08T19:22:17,312 INFO [Time-limited test {}] bucket.BucketCache(365): Instantiating BucketCache with acceptableFactor: 0.95, minFactor: 0.85, extraFreeFactor: 0.1, singleFactor: 0.25, multiFactor: 0.5, memoryFactor: 0.25, useStrongRef: false 2024-11-08T19:22:17,313 INFO [Time-limited test {}] bucket.BucketCache(479): Starting BucketCachePersisterThread 2024-11-08T19:22:17,315 INFO [Time-limited test {}] bucket.BucketCachePersister(38): BucketCachePersister started with interval: 1000 2024-11-08T19:22:17,316 INFO [Thread-10 {}] bucket.BucketCache(1531): Started retrieving bucket cache from file 2024-11-08T19:22:17,316 WARN [Thread-10 {}] bucket.BucketCache(1534): Persistence file missing! It's ok if it's first run after enabling persistent cache. 2024-11-08T19:22:17,325 INFO [Time-limited test {}] bucket.BucketCache(404): Started bucket cache; ioengine=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/8a0ec66d-a62f-b9fa-51db-a1c5c34d17d3/bucket.cache, capacity=400 MB, blockSize=64 KB, writerThreadNum=3, writerQLen=64, persistencePath=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/8a0ec66d-a62f-b9fa-51db-a1c5c34d17d3/bucket.persistence, bucketAllocator=org.apache.hadoop.hbase.io.hfile.bucket.BucketAllocator 2024-11-08T19:22:17,329 INFO [Thread-10 {}] bucket.BucketAllocator(343): Cache totalSize=418148352, buckets=199, bucket capacity=2101248=(4*525312)=(FEWEST_ITEMS_IN_BUCKET*(largest configured bucketcache size)) 2024-11-08T19:22:17,329 INFO [Thread-10 {}] bucket.BucketCache(414): Persistent bucket cache recovery from /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/8a0ec66d-a62f-b9fa-51db-a1c5c34d17d3/bucket.persistence is complete. 2024-11-08T19:22:17,342 DEBUG [Time-limited test {}] mob.MobFileCache(123): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-08T19:22:17,345 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:37371-0x1011be626410001, quorum=127.0.0.1:63355, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-08T19:22:17,351 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:37371-0x1011be626410001, quorum=127.0.0.1:63355, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-08T19:22:17,351 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=37371 2024-11-08T19:22:17,352 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=37371 2024-11-08T19:22:17,353 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=37371 2024-11-08T19:22:17,355 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=37371 2024-11-08T19:22:17,356 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=37371 2024-11-08T19:22:17,380 DEBUG [M:0;692b295ad45c:38529 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;692b295ad45c:38529 2024-11-08T19:22:17,381 INFO [master/692b295ad45c:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/692b295ad45c,38529,1731093736393 2024-11-08T19:22:17,402 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38529-0x1011be626410000, quorum=127.0.0.1:63355, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-08T19:22:17,402 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37371-0x1011be626410001, quorum=127.0.0.1:63355, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-08T19:22:17,404 DEBUG [master/692b295ad45c:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:38529-0x1011be626410000, quorum=127.0.0.1:63355, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/692b295ad45c,38529,1731093736393 2024-11-08T19:22:17,444 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38529-0x1011be626410000, quorum=127.0.0.1:63355, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-08T19:22:17,443 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37371-0x1011be626410001, quorum=127.0.0.1:63355, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-08T19:22:17,444 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37371-0x1011be626410001, quorum=127.0.0.1:63355, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-08T19:22:17,445 DEBUG [master/692b295ad45c:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:38529-0x1011be626410000, quorum=127.0.0.1:63355, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-11-08T19:22:17,447 INFO [master/692b295ad45c:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/692b295ad45c,38529,1731093736393 from backup master directory 2024-11-08T19:22:17,459 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38529-0x1011be626410000, quorum=127.0.0.1:63355, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/692b295ad45c,38529,1731093736393 2024-11-08T19:22:17,459 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37371-0x1011be626410001, quorum=127.0.0.1:63355, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-08T19:22:17,460 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38529-0x1011be626410000, quorum=127.0.0.1:63355, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-08T19:22:17,461 WARN [master/692b295ad45c:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-08T19:22:17,462 INFO [master/692b295ad45c:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=692b295ad45c,38529,1731093736393 2024-11-08T19:22:17,464 INFO [master/692b295ad45c:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating data MemStoreChunkPool with chunk size 2 MB, max count 396, initial count 0 2024-11-08T19:22:17,465 INFO [master/692b295ad45c:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating index MemStoreChunkPool with chunk size 204.80 KB, max count 440, initial count 0 2024-11-08T19:22:17,474 DEBUG [master/692b295ad45c:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/8a0ec66d-a62f-b9fa-51db-a1c5c34d17d3/f4a1b344-4599-9a20-81eb-9d7a968f2df2/hbase.id] with ID: 5b05297d-de05-489b-a1b0-92a852f08601 2024-11-08T19:22:17,475 DEBUG [master/692b295ad45c:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/8a0ec66d-a62f-b9fa-51db-a1c5c34d17d3/f4a1b344-4599-9a20-81eb-9d7a968f2df2/.tmp/hbase.id 2024-11-08T19:22:17,483 DEBUG [master/692b295ad45c:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/8a0ec66d-a62f-b9fa-51db-a1c5c34d17d3/f4a1b344-4599-9a20-81eb-9d7a968f2df2/.tmp/hbase.id]:[file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/8a0ec66d-a62f-b9fa-51db-a1c5c34d17d3/f4a1b344-4599-9a20-81eb-9d7a968f2df2/hbase.id] 2024-11-08T19:22:17,536 DEBUG [master/692b295ad45c:0:becomeActiveMaster {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-08T19:22:17,542 INFO [master/692b295ad45c:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem. 2024-11-08T19:22:17,561 INFO [master/692b295ad45c:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 17ms. 2024-11-08T19:22:17,575 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37371-0x1011be626410001, quorum=127.0.0.1:63355, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-08T19:22:17,575 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38529-0x1011be626410000, quorum=127.0.0.1:63355, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-08T19:22:17,600 INFO [master/692b295ad45c:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-08T19:22:17,603 INFO [master/692b295ad45c:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-11-08T19:22:17,644 DEBUG [master/692b295ad45c:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(244): No decryptEncryptedDataEncryptionKey method in DFSClient, should be hadoop version with HDFS-12396 java.lang.NoSuchMethodException: org.apache.hadoop.hdfs.DFSClient.decryptEncryptedDataEncryptionKey(org.apache.hadoop.fs.FileEncryptionInfo) at java.lang.Class.getDeclaredMethod(Class.java:2675) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelperWithoutHDFS12396(FanOutOneBlockAsyncDFSOutputSaslHelper.java:183) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelper(FanOutOneBlockAsyncDFSOutputSaslHelper.java:242) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.(FanOutOneBlockAsyncDFSOutputSaslHelper.java:253) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.lang.Class.forName0(Native Method) ~[?:?] at java.lang.Class.forName(Class.java:375) ~[?:?] at org.apache.hadoop.hbase.wal.AsyncFSWALProvider.load(AsyncFSWALProvider.java:150) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.getProviderClass(WALFactory.java:174) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:262) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:231) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegion.create(MasterRegion.java:400) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegionFactory.create(MasterRegionFactory.java:135) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.finishActiveMasterInitialization(HMaster.java:1003) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.startActiveMasterManager(HMaster.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.lambda$run$0(HMaster.java:613) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.lambda$tracedRunnable$2(TraceUtil.java:155) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T19:22:17,649 INFO [master/692b295ad45c:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-08T19:22:17,709 INFO [master/692b295ad45c:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/8a0ec66d-a62f-b9fa-51db-a1c5c34d17d3/f4a1b344-4599-9a20-81eb-9d7a968f2df2/MasterData/data/master/store 2024-11-08T19:22:17,734 INFO [master/692b295ad45c:0:becomeActiveMaster {}] throttle.StoreHotnessProtector(112): StoreHotnessProtector is disabled. Set hbase.region.store.parallel.put.limit > 0 to enable, which may help mitigate load under heavy write pressure. 2024-11-08T19:22:17,738 DEBUG [master/692b295ad45c:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-08T19:22:17,740 DEBUG [master/692b295ad45c:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-08T19:22:17,741 INFO [master/692b295ad45c:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-08T19:22:17,741 DEBUG [master/692b295ad45c:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-08T19:22:17,744 DEBUG [master/692b295ad45c:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-08T19:22:17,744 DEBUG [master/692b295ad45c:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-08T19:22:17,744 INFO [master/692b295ad45c:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-08T19:22:17,745 DEBUG [master/692b295ad45c:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1731093737740Disabling compacts and flushes for region at 1731093737740Disabling writes for close at 1731093737744 (+4 ms)Writing region close event to WAL at 1731093737744Closed at 1731093737744 2024-11-08T19:22:17,746 WARN [master/692b295ad45c:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/8a0ec66d-a62f-b9fa-51db-a1c5c34d17d3/f4a1b344-4599-9a20-81eb-9d7a968f2df2/MasterData/data/master/store/.initializing 2024-11-08T19:22:17,746 DEBUG [master/692b295ad45c:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/8a0ec66d-a62f-b9fa-51db-a1c5c34d17d3/f4a1b344-4599-9a20-81eb-9d7a968f2df2/MasterData/WALs/692b295ad45c,38529,1731093736393 2024-11-08T19:22:17,759 INFO [master/692b295ad45c:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-08T19:22:17,775 INFO [master/692b295ad45c:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=692b295ad45c%2C38529%2C1731093736393, suffix=, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/8a0ec66d-a62f-b9fa-51db-a1c5c34d17d3/f4a1b344-4599-9a20-81eb-9d7a968f2df2/MasterData/WALs/692b295ad45c,38529,1731093736393, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/8a0ec66d-a62f-b9fa-51db-a1c5c34d17d3/f4a1b344-4599-9a20-81eb-9d7a968f2df2/MasterData/oldWALs, maxLogs=10 2024-11-08T19:22:17,818 INFO [master/692b295ad45c:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/8a0ec66d-a62f-b9fa-51db-a1c5c34d17d3/f4a1b344-4599-9a20-81eb-9d7a968f2df2/MasterData/WALs/692b295ad45c,38529,1731093736393/692b295ad45c%2C38529%2C1731093736393.1731093737779 2024-11-08T19:22:17,818 DEBUG [master/692b295ad45c:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-11-08T19:22:17,819 DEBUG [master/692b295ad45c:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-11-08T19:22:17,820 DEBUG [master/692b295ad45c:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-08T19:22:17,824 DEBUG [master/692b295ad45c:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-11-08T19:22:17,825 DEBUG [master/692b295ad45c:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-11-08T19:22:17,866 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=true, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-11-08T19:22:17,899 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-11-08T19:22:17,904 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T19:22:17,906 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-08T19:22:17,907 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=true, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-11-08T19:22:17,915 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-11-08T19:22:17,915 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T19:22:17,917 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-08T19:22:17,918 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=true, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-11-08T19:22:17,923 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-11-08T19:22:17,924 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T19:22:17,924 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-08T19:22:17,925 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=true, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-11-08T19:22:17,932 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-11-08T19:22:17,932 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T19:22:17,933 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-08T19:22:17,933 DEBUG [master/692b295ad45c:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682 2024-11-08T19:22:17,938 DEBUG [master/692b295ad45c:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/8a0ec66d-a62f-b9fa-51db-a1c5c34d17d3/f4a1b344-4599-9a20-81eb-9d7a968f2df2/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-08T19:22:17,940 DEBUG [master/692b295ad45c:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/8a0ec66d-a62f-b9fa-51db-a1c5c34d17d3/f4a1b344-4599-9a20-81eb-9d7a968f2df2/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-08T19:22:17,944 DEBUG [master/692b295ad45c:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682 2024-11-08T19:22:17,945 DEBUG [master/692b295ad45c:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682 2024-11-08T19:22:17,948 DEBUG [master/692b295ad45c:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-08T19:22:17,951 DEBUG [master/692b295ad45c:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-11-08T19:22:17,957 DEBUG [master/692b295ad45c:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/8a0ec66d-a62f-b9fa-51db-a1c5c34d17d3/f4a1b344-4599-9a20-81eb-9d7a968f2df2/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-08T19:22:17,958 INFO [master/692b295ad45c:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=73711289, jitterRate=0.09838379919528961}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-08T19:22:17,965 DEBUG [master/692b295ad45c:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682: Writing region info on filesystem at 1731093737838Initializing all the Stores at 1731093737839 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731093737839Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731093737840 (+1 ms)Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731093737840Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731093737840Cleaning up temporary data from old regions at 1731093737945 (+105 ms)Region opened successfully at 1731093737965 (+20 ms) 2024-11-08T19:22:17,967 INFO [master/692b295ad45c:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-11-08T19:22:17,997 DEBUG [master/692b295ad45c:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@19819e3a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=692b295ad45c/172.17.0.2:0 2024-11-08T19:22:18,028 INFO [master/692b295ad45c:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating... 2024-11-08T19:22:18,039 INFO [master/692b295ad45c:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5 2024-11-08T19:22:18,039 INFO [master/692b295ad45c:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50 2024-11-08T19:22:18,042 INFO [master/692b295ad45c:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-08T19:22:18,043 INFO [master/692b295ad45c:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-08T19:22:18,048 INFO [master/692b295ad45c:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 5 msec 2024-11-08T19:22:18,048 INFO [master/692b295ad45c:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-11-08T19:22:18,072 INFO [master/692b295ad45c:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2024-11-08T19:22:18,081 DEBUG [master/692b295ad45c:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:38529-0x1011be626410000, quorum=127.0.0.1:63355, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error) 2024-11-08T19:22:18,159 DEBUG [master/692b295ad45c:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false 2024-11-08T19:22:18,162 INFO [master/692b295ad45c:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2024-11-08T19:22:18,164 DEBUG [master/692b295ad45c:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:38529-0x1011be626410000, quorum=127.0.0.1:63355, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error) 2024-11-08T19:22:18,247 DEBUG [master/692b295ad45c:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false 2024-11-08T19:22:18,251 INFO [master/692b295ad45c:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2024-11-08T19:22:18,256 DEBUG [master/692b295ad45c:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:38529-0x1011be626410000, quorum=127.0.0.1:63355, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error) 2024-11-08T19:22:18,285 DEBUG [master/692b295ad45c:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false 2024-11-08T19:22:18,290 DEBUG [master/692b295ad45c:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:38529-0x1011be626410000, quorum=127.0.0.1:63355, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error) 2024-11-08T19:22:18,301 DEBUG [master/692b295ad45c:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false 2024-11-08T19:22:18,317 DEBUG [master/692b295ad45c:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:38529-0x1011be626410000, quorum=127.0.0.1:63355, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error) 2024-11-08T19:22:18,327 DEBUG [master/692b295ad45c:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false 2024-11-08T19:22:18,348 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38529-0x1011be626410000, quorum=127.0.0.1:63355, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-08T19:22:18,348 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37371-0x1011be626410001, quorum=127.0.0.1:63355, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-08T19:22:18,348 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38529-0x1011be626410000, quorum=127.0.0.1:63355, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-08T19:22:18,349 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37371-0x1011be626410001, quorum=127.0.0.1:63355, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-08T19:22:18,351 INFO [master/692b295ad45c:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=692b295ad45c,38529,1731093736393, sessionid=0x1011be626410000, setting cluster-up flag (Was=false) 2024-11-08T19:22:18,380 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37371-0x1011be626410001, quorum=127.0.0.1:63355, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-08T19:22:18,380 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38529-0x1011be626410000, quorum=127.0.0.1:63355, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-08T19:22:18,412 DEBUG [master/692b295ad45c:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort 2024-11-08T19:22:18,416 DEBUG [master/692b295ad45c:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=692b295ad45c,38529,1731093736393 2024-11-08T19:22:18,438 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38529-0x1011be626410000, quorum=127.0.0.1:63355, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-08T19:22:18,438 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37371-0x1011be626410001, quorum=127.0.0.1:63355, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-08T19:22:18,470 DEBUG [master/692b295ad45c:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort 2024-11-08T19:22:18,475 DEBUG [master/692b295ad45c:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=692b295ad45c,38529,1731093736393 2024-11-08T19:22:18,486 INFO [master/692b295ad45c:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/8a0ec66d-a62f-b9fa-51db-a1c5c34d17d3/f4a1b344-4599-9a20-81eb-9d7a968f2df2/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again 2024-11-08T19:22:18,561 INFO [RS:0;692b295ad45c:37371 {}] regionserver.HRegionServer(746): ClusterId : 5b05297d-de05-489b-a1b0-92a852f08601 2024-11-08T19:22:18,563 DEBUG [RS:0;692b295ad45c:37371 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-08T19:22:18,563 DEBUG [master/692b295ad45c:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-08T19:22:18,577 DEBUG [RS:0;692b295ad45c:37371 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-08T19:22:18,577 DEBUG [RS:0;692b295ad45c:37371 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-08T19:22:18,577 INFO [master/692b295ad45c:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2 2024-11-08T19:22:18,586 INFO [master/692b295ad45c:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2024-11-08T19:22:18,592 DEBUG [RS:0;692b295ad45c:37371 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-08T19:22:18,593 DEBUG [RS:0;692b295ad45c:37371 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3ccf5d76, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=692b295ad45c/172.17.0.2:0 2024-11-08T19:22:18,593 DEBUG [master/692b295ad45c:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: 692b295ad45c,38529,1731093736393 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2024-11-08T19:22:18,603 DEBUG [master/692b295ad45c:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/692b295ad45c:0, corePoolSize=5, maxPoolSize=5 2024-11-08T19:22:18,603 DEBUG [master/692b295ad45c:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/692b295ad45c:0, corePoolSize=5, maxPoolSize=5 2024-11-08T19:22:18,603 DEBUG [master/692b295ad45c:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/692b295ad45c:0, corePoolSize=5, maxPoolSize=5 2024-11-08T19:22:18,604 DEBUG [master/692b295ad45c:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/692b295ad45c:0, corePoolSize=5, maxPoolSize=5 2024-11-08T19:22:18,604 DEBUG [master/692b295ad45c:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/692b295ad45c:0, corePoolSize=10, maxPoolSize=10 2024-11-08T19:22:18,604 DEBUG [master/692b295ad45c:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/692b295ad45c:0, corePoolSize=1, maxPoolSize=1 2024-11-08T19:22:18,605 DEBUG [master/692b295ad45c:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/692b295ad45c:0, corePoolSize=2, maxPoolSize=2 2024-11-08T19:22:18,605 DEBUG [master/692b295ad45c:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/692b295ad45c:0, corePoolSize=1, maxPoolSize=1 2024-11-08T19:22:18,606 INFO [master/692b295ad45c:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731093768606 2024-11-08T19:22:18,608 DEBUG [RS:0;692b295ad45c:37371 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;692b295ad45c:37371 2024-11-08T19:22:18,609 INFO [master/692b295ad45c:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2024-11-08T19:22:18,610 INFO [master/692b295ad45c:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2024-11-08T19:22:18,610 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-08T19:22:18,611 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region 2024-11-08T19:22:18,612 INFO [RS:0;692b295ad45c:37371 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-08T19:22:18,612 INFO [RS:0;692b295ad45c:37371 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-08T19:22:18,612 DEBUG [RS:0;692b295ad45c:37371 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-08T19:22:18,613 INFO [master/692b295ad45c:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2024-11-08T19:22:18,614 INFO [master/692b295ad45c:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2024-11-08T19:22:18,614 INFO [master/692b295ad45c:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2024-11-08T19:22:18,614 INFO [master/692b295ad45c:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2024-11-08T19:22:18,615 INFO [RS:0;692b295ad45c:37371 {}] regionserver.HRegionServer(2659): reportForDuty to master=692b295ad45c,38529,1731093736393 with port=37371, startcode=1731093737227 2024-11-08T19:22:18,616 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T19:22:18,616 INFO [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-11-08T19:22:18,617 INFO [master/692b295ad45c:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-08T19:22:18,621 INFO [master/692b295ad45c:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2024-11-08T19:22:18,622 INFO [master/692b295ad45c:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2024-11-08T19:22:18,622 INFO [master/692b295ad45c:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2024-11-08T19:22:18,625 INFO [master/692b295ad45c:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2024-11-08T19:22:18,625 INFO [master/692b295ad45c:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2024-11-08T19:22:18,627 DEBUG [master/692b295ad45c:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/692b295ad45c:0:becomeActiveMaster-HFileCleaner.large.0-1731093738626,5,FailOnTimeoutGroup] 2024-11-08T19:22:18,628 DEBUG [master/692b295ad45c:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/692b295ad45c:0:becomeActiveMaster-HFileCleaner.small.0-1731093738627,5,FailOnTimeoutGroup] 2024-11-08T19:22:18,628 INFO [master/692b295ad45c:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-08T19:22:18,628 INFO [master/692b295ad45c:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2024-11-08T19:22:18,629 INFO [master/692b295ad45c:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2024-11-08T19:22:18,630 INFO [master/692b295ad45c:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2024-11-08T19:22:18,630 DEBUG [RS:0;692b295ad45c:37371 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-08T19:22:18,643 INFO [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/8a0ec66d-a62f-b9fa-51db-a1c5c34d17d3/f4a1b344-4599-9a20-81eb-9d7a968f2df2/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321 2024-11-08T19:22:18,644 INFO [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/8a0ec66d-a62f-b9fa-51db-a1c5c34d17d3/f4a1b344-4599-9a20-81eb-9d7a968f2df2 2024-11-08T19:22:18,653 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-08T19:22:18,657 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=true, for column family info of region 1588230740 2024-11-08T19:22:18,664 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-08T19:22:18,664 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T19:22:18,665 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-08T19:22:18,665 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=true, for column family ns of region 1588230740 2024-11-08T19:22:18,670 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-08T19:22:18,670 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T19:22:18,675 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-08T19:22:18,675 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=true, for column family rep_barrier of region 1588230740 2024-11-08T19:22:18,680 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-08T19:22:18,680 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T19:22:18,681 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-08T19:22:18,681 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=true, for column family table of region 1588230740 2024-11-08T19:22:18,685 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-08T19:22:18,686 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T19:22:18,686 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-08T19:22:18,687 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-08T19:22:18,688 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/8a0ec66d-a62f-b9fa-51db-a1c5c34d17d3/f4a1b344-4599-9a20-81eb-9d7a968f2df2/data/hbase/meta/1588230740 2024-11-08T19:22:18,688 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/8a0ec66d-a62f-b9fa-51db-a1c5c34d17d3/f4a1b344-4599-9a20-81eb-9d7a968f2df2/data/hbase/meta/1588230740 2024-11-08T19:22:18,689 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-08T19:22:18,689 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-08T19:22:18,689 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-08T19:22:18,690 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-08T19:22:18,704 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/8a0ec66d-a62f-b9fa-51db-a1c5c34d17d3/f4a1b344-4599-9a20-81eb-9d7a968f2df2/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-08T19:22:18,705 INFO [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=64113759, jitterRate=-0.044630542397499084}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-08T19:22:18,705 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40087, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=RegionServerStatusService 2024-11-08T19:22:18,710 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740: Writing region info on filesystem at 1731093738653Initializing all the Stores at 1731093738653Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731093738653Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731093738657 (+4 ms)Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731093738657Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731093738657Cleaning up temporary data from old regions at 1731093738689 (+32 ms)Region opened successfully at 1731093738710 (+21 ms) 2024-11-08T19:22:18,711 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-08T19:22:18,711 INFO [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-08T19:22:18,712 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-08T19:22:18,712 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-08T19:22:18,712 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-08T19:22:18,714 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=38529 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 692b295ad45c,37371,1731093737227 2024-11-08T19:22:18,716 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=38529 {}] master.ServerManager(517): Registering regionserver=692b295ad45c,37371,1731093737227 2024-11-08T19:22:18,717 INFO [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-08T19:22:18,717 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1731093738711Disabling compacts and flushes for region at 1731093738711Disabling writes for close at 1731093738712 (+1 ms)Writing region close event to WAL at 1731093738716 (+4 ms)Closed at 1731093738716 2024-11-08T19:22:18,720 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-08T19:22:18,720 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta 2024-11-08T19:22:18,729 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-11-08T19:22:18,734 DEBUG [RS:0;692b295ad45c:37371 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/8a0ec66d-a62f-b9fa-51db-a1c5c34d17d3/f4a1b344-4599-9a20-81eb-9d7a968f2df2 2024-11-08T19:22:18,734 DEBUG [RS:0;692b295ad45c:37371 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=file:/ 2024-11-08T19:22:18,734 DEBUG [RS:0;692b295ad45c:37371 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-08T19:22:18,738 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-08T19:22:18,741 INFO [PEWorker-5 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2024-11-08T19:22:18,754 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38529-0x1011be626410000, quorum=127.0.0.1:63355, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-08T19:22:18,755 DEBUG [RS:0;692b295ad45c:37371 {}] zookeeper.ZKUtil(111): regionserver:37371-0x1011be626410001, quorum=127.0.0.1:63355, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/692b295ad45c,37371,1731093737227 2024-11-08T19:22:18,755 WARN [RS:0;692b295ad45c:37371 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-08T19:22:18,755 INFO [RS:0;692b295ad45c:37371 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-08T19:22:18,755 DEBUG [RS:0;692b295ad45c:37371 {}] regionserver.HRegionServer(1793): logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/8a0ec66d-a62f-b9fa-51db-a1c5c34d17d3/f4a1b344-4599-9a20-81eb-9d7a968f2df2/WALs/692b295ad45c,37371,1731093737227 2024-11-08T19:22:18,758 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [692b295ad45c,37371,1731093737227] 2024-11-08T19:22:18,796 INFO [RS:0;692b295ad45c:37371 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-08T19:22:18,820 INFO [RS:0;692b295ad45c:37371 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-08T19:22:18,828 INFO [RS:0;692b295ad45c:37371 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-08T19:22:18,828 INFO [RS:0;692b295ad45c:37371 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-08T19:22:18,830 INFO [RS:0;692b295ad45c:37371 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-08T19:22:18,837 INFO [RS:0;692b295ad45c:37371 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-08T19:22:18,839 INFO [RS:0;692b295ad45c:37371 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-08T19:22:18,840 DEBUG [RS:0;692b295ad45c:37371 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/692b295ad45c:0, corePoolSize=1, maxPoolSize=1 2024-11-08T19:22:18,840 DEBUG [RS:0;692b295ad45c:37371 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/692b295ad45c:0, corePoolSize=1, maxPoolSize=1 2024-11-08T19:22:18,840 DEBUG [RS:0;692b295ad45c:37371 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/692b295ad45c:0, corePoolSize=1, maxPoolSize=1 2024-11-08T19:22:18,841 DEBUG [RS:0;692b295ad45c:37371 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/692b295ad45c:0, corePoolSize=1, maxPoolSize=1 2024-11-08T19:22:18,841 DEBUG [RS:0;692b295ad45c:37371 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/692b295ad45c:0, corePoolSize=1, maxPoolSize=1 2024-11-08T19:22:18,841 DEBUG [RS:0;692b295ad45c:37371 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/692b295ad45c:0, corePoolSize=2, maxPoolSize=2 2024-11-08T19:22:18,841 DEBUG [RS:0;692b295ad45c:37371 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/692b295ad45c:0, corePoolSize=1, maxPoolSize=1 2024-11-08T19:22:18,841 DEBUG [RS:0;692b295ad45c:37371 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/692b295ad45c:0, corePoolSize=1, maxPoolSize=1 2024-11-08T19:22:18,842 DEBUG [RS:0;692b295ad45c:37371 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/692b295ad45c:0, corePoolSize=1, maxPoolSize=1 2024-11-08T19:22:18,842 DEBUG [RS:0;692b295ad45c:37371 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/692b295ad45c:0, corePoolSize=1, maxPoolSize=1 2024-11-08T19:22:18,842 DEBUG [RS:0;692b295ad45c:37371 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/692b295ad45c:0, corePoolSize=1, maxPoolSize=1 2024-11-08T19:22:18,842 DEBUG [RS:0;692b295ad45c:37371 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/692b295ad45c:0, corePoolSize=1, maxPoolSize=1 2024-11-08T19:22:18,842 DEBUG [RS:0;692b295ad45c:37371 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/692b295ad45c:0, corePoolSize=3, maxPoolSize=3 2024-11-08T19:22:18,842 DEBUG [RS:0;692b295ad45c:37371 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/692b295ad45c:0, corePoolSize=3, maxPoolSize=3 2024-11-08T19:22:18,844 INFO [RS:0;692b295ad45c:37371 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-08T19:22:18,844 INFO [RS:0;692b295ad45c:37371 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-08T19:22:18,844 INFO [RS:0;692b295ad45c:37371 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-08T19:22:18,844 INFO [RS:0;692b295ad45c:37371 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-08T19:22:18,844 INFO [RS:0;692b295ad45c:37371 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-08T19:22:18,845 INFO [RS:0;692b295ad45c:37371 {}] hbase.ChoreService(168): Chore ScheduledChore name=692b295ad45c,37371,1731093737227-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-08T19:22:18,874 INFO [RS:0;692b295ad45c:37371 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-08T19:22:18,877 INFO [RS:0;692b295ad45c:37371 {}] hbase.ChoreService(168): Chore ScheduledChore name=692b295ad45c,37371,1731093737227-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-08T19:22:18,877 INFO [RS:0;692b295ad45c:37371 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-08T19:22:18,877 INFO [RS:0;692b295ad45c:37371 {}] regionserver.Replication(171): 692b295ad45c,37371,1731093737227 started 2024-11-08T19:22:18,892 WARN [692b295ad45c:38529 {}] assignment.AssignmentManager(2443): No servers available; cannot place 1 unassigned regions. 2024-11-08T19:22:18,905 INFO [RS:0;692b295ad45c:37371 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-08T19:22:18,905 INFO [RS:0;692b295ad45c:37371 {}] regionserver.HRegionServer(1482): Serving as 692b295ad45c,37371,1731093737227, RpcServer on 692b295ad45c/172.17.0.2:37371, sessionid=0x1011be626410001 2024-11-08T19:22:18,906 DEBUG [RS:0;692b295ad45c:37371 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-08T19:22:18,906 DEBUG [RS:0;692b295ad45c:37371 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 692b295ad45c,37371,1731093737227 2024-11-08T19:22:18,907 DEBUG [RS:0;692b295ad45c:37371 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '692b295ad45c,37371,1731093737227' 2024-11-08T19:22:18,907 DEBUG [RS:0;692b295ad45c:37371 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-08T19:22:18,908 DEBUG [RS:0;692b295ad45c:37371 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-08T19:22:18,909 DEBUG [RS:0;692b295ad45c:37371 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-08T19:22:18,909 DEBUG [RS:0;692b295ad45c:37371 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-08T19:22:18,909 DEBUG [RS:0;692b295ad45c:37371 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 692b295ad45c,37371,1731093737227 2024-11-08T19:22:18,909 DEBUG [RS:0;692b295ad45c:37371 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '692b295ad45c,37371,1731093737227' 2024-11-08T19:22:18,910 DEBUG [RS:0;692b295ad45c:37371 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-08T19:22:18,911 DEBUG [RS:0;692b295ad45c:37371 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-08T19:22:18,911 DEBUG [RS:0;692b295ad45c:37371 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-08T19:22:18,911 INFO [RS:0;692b295ad45c:37371 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-08T19:22:18,912 INFO [RS:0;692b295ad45c:37371 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-08T19:22:19,016 INFO [RS:0;692b295ad45c:37371 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-08T19:22:19,017 INFO [RS:0;692b295ad45c:37371 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=692b295ad45c%2C37371%2C1731093737227, suffix=, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/8a0ec66d-a62f-b9fa-51db-a1c5c34d17d3/f4a1b344-4599-9a20-81eb-9d7a968f2df2/WALs/692b295ad45c,37371,1731093737227, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/8a0ec66d-a62f-b9fa-51db-a1c5c34d17d3/f4a1b344-4599-9a20-81eb-9d7a968f2df2/oldWALs, maxLogs=55 2024-11-08T19:22:19,045 INFO [RS:0;692b295ad45c:37371 {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/8a0ec66d-a62f-b9fa-51db-a1c5c34d17d3/f4a1b344-4599-9a20-81eb-9d7a968f2df2/WALs/692b295ad45c,37371,1731093737227/692b295ad45c%2C37371%2C1731093737227.1731093739020 2024-11-08T19:22:19,048 DEBUG [RS:0;692b295ad45c:37371 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-11-08T19:22:19,145 DEBUG [692b295ad45c:38529 {}] assignment.AssignmentManager(2464): Processing assignQueue; systemServersCount=1, allServersCount=1 2024-11-08T19:22:19,161 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=692b295ad45c,37371,1731093737227 2024-11-08T19:22:19,169 INFO [PEWorker-2 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 692b295ad45c,37371,1731093737227, state=OPENING 2024-11-08T19:22:19,227 DEBUG [PEWorker-2 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2024-11-08T19:22:19,238 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37371-0x1011be626410001, quorum=127.0.0.1:63355, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-08T19:22:19,238 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38529-0x1011be626410000, quorum=127.0.0.1:63355, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-08T19:22:19,239 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-08T19:22:19,239 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-08T19:22:19,242 DEBUG [PEWorker-2 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-08T19:22:19,245 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=692b295ad45c,37371,1731093737227}] 2024-11-08T19:22:19,423 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-08T19:22:19,428 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38537, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-08T19:22:19,441 INFO [RS_OPEN_META-regionserver/692b295ad45c:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-11-08T19:22:19,442 INFO [RS_OPEN_META-regionserver/692b295ad45c:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-08T19:22:19,443 INFO [RS_OPEN_META-regionserver/692b295ad45c:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-11-08T19:22:19,444 INFO [RS_OPEN_META-regionserver/692b295ad45c:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=692b295ad45c%2C37371%2C1731093737227.meta, suffix=.meta, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/8a0ec66d-a62f-b9fa-51db-a1c5c34d17d3/f4a1b344-4599-9a20-81eb-9d7a968f2df2/WALs/692b295ad45c,37371,1731093737227, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/8a0ec66d-a62f-b9fa-51db-a1c5c34d17d3/f4a1b344-4599-9a20-81eb-9d7a968f2df2/oldWALs, maxLogs=55 2024-11-08T19:22:19,468 INFO [RS_OPEN_META-regionserver/692b295ad45c:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/8a0ec66d-a62f-b9fa-51db-a1c5c34d17d3/f4a1b344-4599-9a20-81eb-9d7a968f2df2/WALs/692b295ad45c,37371,1731093737227/692b295ad45c%2C37371%2C1731093737227.meta.1731093739446.meta 2024-11-08T19:22:19,469 DEBUG [RS_OPEN_META-regionserver/692b295ad45c:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-11-08T19:22:19,469 DEBUG [RS_OPEN_META-regionserver/692b295ad45c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-11-08T19:22:19,472 DEBUG [RS_OPEN_META-regionserver/692b295ad45c:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-11-08T19:22:19,475 DEBUG [RS_OPEN_META-regionserver/692b295ad45c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-11-08T19:22:19,482 INFO [RS_OPEN_META-regionserver/692b295ad45c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-11-08T19:22:19,489 DEBUG [RS_OPEN_META-regionserver/692b295ad45c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-11-08T19:22:19,490 DEBUG [RS_OPEN_META-regionserver/692b295ad45c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-08T19:22:19,490 DEBUG [RS_OPEN_META-regionserver/692b295ad45c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-11-08T19:22:19,491 DEBUG [RS_OPEN_META-regionserver/692b295ad45c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-11-08T19:22:19,496 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=true, for column family info of region 1588230740 2024-11-08T19:22:19,497 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-08T19:22:19,498 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T19:22:19,498 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-08T19:22:19,498 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=true, for column family ns of region 1588230740 2024-11-08T19:22:19,499 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-08T19:22:19,499 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T19:22:19,499 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-08T19:22:19,500 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=true, for column family rep_barrier of region 1588230740 2024-11-08T19:22:19,501 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-08T19:22:19,501 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T19:22:19,501 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-08T19:22:19,501 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=true, for column family table of region 1588230740 2024-11-08T19:22:19,503 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-08T19:22:19,503 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T19:22:19,503 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-08T19:22:19,504 DEBUG [RS_OPEN_META-regionserver/692b295ad45c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-08T19:22:19,505 DEBUG [RS_OPEN_META-regionserver/692b295ad45c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/8a0ec66d-a62f-b9fa-51db-a1c5c34d17d3/f4a1b344-4599-9a20-81eb-9d7a968f2df2/data/hbase/meta/1588230740 2024-11-08T19:22:19,506 DEBUG [RS_OPEN_META-regionserver/692b295ad45c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/8a0ec66d-a62f-b9fa-51db-a1c5c34d17d3/f4a1b344-4599-9a20-81eb-9d7a968f2df2/data/hbase/meta/1588230740 2024-11-08T19:22:19,506 DEBUG [RS_OPEN_META-regionserver/692b295ad45c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-08T19:22:19,506 DEBUG [RS_OPEN_META-regionserver/692b295ad45c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-08T19:22:19,507 DEBUG [RS_OPEN_META-regionserver/692b295ad45c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-08T19:22:19,508 DEBUG [RS_OPEN_META-regionserver/692b295ad45c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-08T19:22:19,509 INFO [RS_OPEN_META-regionserver/692b295ad45c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=69988372, jitterRate=0.04290801286697388}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-08T19:22:19,510 DEBUG [RS_OPEN_META-regionserver/692b295ad45c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-11-08T19:22:19,512 DEBUG [RS_OPEN_META-regionserver/692b295ad45c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1731093739492Writing region info on filesystem at 1731093739492Initializing all the Stores at 1731093739492Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731093739493 (+1 ms)Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731093739496 (+3 ms)Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731093739496Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731093739496Cleaning up temporary data from old regions at 1731093739506 (+10 ms)Running coprocessor post-open hooks at 1731093739510 (+4 ms)Region opened successfully at 1731093739511 (+1 ms) 2024-11-08T19:22:19,518 INFO [RS_OPEN_META-regionserver/692b295ad45c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1731093739412 2024-11-08T19:22:19,530 DEBUG [RS_OPEN_META-regionserver/692b295ad45c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-11-08T19:22:19,531 INFO [RS_OPEN_META-regionserver/692b295ad45c:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-11-08T19:22:19,532 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=692b295ad45c,37371,1731093737227 2024-11-08T19:22:19,534 INFO [PEWorker-4 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 692b295ad45c,37371,1731093737227, state=OPEN 2024-11-08T19:22:19,664 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37371-0x1011be626410001, quorum=127.0.0.1:63355, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-08T19:22:19,664 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38529-0x1011be626410000, quorum=127.0.0.1:63355, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-08T19:22:19,664 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-08T19:22:19,664 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-08T19:22:19,665 DEBUG [PEWorker-4 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=692b295ad45c,37371,1731093737227 2024-11-08T19:22:19,670 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2 2024-11-08T19:22:19,670 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=692b295ad45c,37371,1731093737227 in 421 msec 2024-11-08T19:22:19,677 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1 2024-11-08T19:22:19,677 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 944 msec 2024-11-08T19:22:19,680 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-08T19:22:19,680 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces 2024-11-08T19:22:19,706 DEBUG [PEWorker-1 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-08T19:22:19,709 DEBUG [PEWorker-1 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=692b295ad45c,37371,1731093737227, seqNum=-1] 2024-11-08T19:22:19,734 DEBUG [PEWorker-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-08T19:22:19,737 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51721, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-08T19:22:19,760 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 1.2370 sec 2024-11-08T19:22:19,761 INFO [master/692b295ad45c:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1731093739760, completionTime=-1 2024-11-08T19:22:19,764 INFO [master/692b295ad45c:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=1; waited=0ms, expected min=1 server(s), max=1 server(s), master is running 2024-11-08T19:22:19,764 DEBUG [master/692b295ad45c:0:becomeActiveMaster {}] assignment.AssignmentManager(1756): Joining cluster... 2024-11-08T19:22:19,799 INFO [master/692b295ad45c:0:becomeActiveMaster {}] assignment.AssignmentManager(1768): Number of RegionServers=1 2024-11-08T19:22:19,799 INFO [master/692b295ad45c:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731093799799 2024-11-08T19:22:19,799 INFO [master/692b295ad45c:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731093859799 2024-11-08T19:22:19,799 INFO [master/692b295ad45c:0:becomeActiveMaster {}] assignment.AssignmentManager(1775): Joined the cluster in 34 msec 2024-11-08T19:22:19,803 INFO [master/692b295ad45c:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=692b295ad45c,38529,1731093736393-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-08T19:22:19,804 INFO [master/692b295ad45c:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=692b295ad45c,38529,1731093736393-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-08T19:22:19,804 INFO [master/692b295ad45c:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=692b295ad45c,38529,1731093736393-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-08T19:22:19,806 INFO [master/692b295ad45c:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-692b295ad45c:38529, period=300000, unit=MILLISECONDS is enabled. 2024-11-08T19:22:19,806 INFO [master/692b295ad45c:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2024-11-08T19:22:19,807 INFO [master/692b295ad45c:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled. 2024-11-08T19:22:19,822 DEBUG [master/692b295ad45c:0.Chore.1 {}] janitor.CatalogJanitor(180): 2024-11-08T19:22:19,842 INFO [master/692b295ad45c:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 2.380sec 2024-11-08T19:22:19,844 INFO [master/692b295ad45c:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2024-11-08T19:22:19,845 INFO [master/692b295ad45c:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2024-11-08T19:22:19,846 INFO [master/692b295ad45c:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2024-11-08T19:22:19,847 INFO [master/692b295ad45c:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2024-11-08T19:22:19,847 INFO [master/692b295ad45c:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2024-11-08T19:22:19,848 INFO [master/692b295ad45c:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=692b295ad45c,38529,1731093736393-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-08T19:22:19,849 INFO [master/692b295ad45c:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=692b295ad45c,38529,1731093736393-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2024-11-08T19:22:19,854 DEBUG [master/692b295ad45c:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds 2024-11-08T19:22:19,855 INFO [master/692b295ad45c:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2024-11-08T19:22:19,855 INFO [master/692b295ad45c:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=692b295ad45c,38529,1731093736393-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-08T19:22:19,871 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@195e248f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T19:22:19,873 DEBUG [Time-limited test {}] nio.NioEventLoop(110): -Dio.netty.noKeySetOptimization: false 2024-11-08T19:22:19,873 DEBUG [Time-limited test {}] nio.NioEventLoop(111): -Dio.netty.selectorAutoRebuildThreshold: 512 2024-11-08T19:22:19,878 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 692b295ad45c,38529,-1 for getting cluster id 2024-11-08T19:22:19,880 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-08T19:22:19,889 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '5b05297d-de05-489b-a1b0-92a852f08601' 2024-11-08T19:22:19,893 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-08T19:22:19,893 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "5b05297d-de05-489b-a1b0-92a852f08601" 2024-11-08T19:22:19,894 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3791578b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T19:22:19,894 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [692b295ad45c,38529,-1] 2024-11-08T19:22:19,897 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-08T19:22:19,899 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T19:22:19,902 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44092, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-08T19:22:19,906 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6a63dbd3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T19:22:19,907 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-08T19:22:19,915 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=692b295ad45c,37371,1731093737227, seqNum=-1] 2024-11-08T19:22:19,915 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-08T19:22:19,918 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37512, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-08T19:22:19,943 INFO [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=692b295ad45c,38529,1731093736393 2024-11-08T19:22:19,948 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-08T19:22:19,954 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncConnectionImpl(321): The fetched master address is 692b295ad45c,38529,1731093736393 2024-11-08T19:22:19,973 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@1e620874 2024-11-08T19:22:19,975 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-08T19:22:19,979 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44106, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-08T19:22:19,992 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38529 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'table1', {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-08T19:22:20,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38529 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=table1 2024-11-08T19:22:20,004 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=table1 execute state=CREATE_TABLE_PRE_OPERATION 2024-11-08T19:22:20,008 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38529 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "table1" procId is: 4 2024-11-08T19:22:20,008 DEBUG [PEWorker-2 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T19:22:20,010 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=table1 execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-08T19:22:20,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38529 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-08T19:22:20,035 INFO [RegionOpenAndInit-table1-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => daec8622ff352a8d55744140f13bcdc5, NAME => 'table1,,1731093739981.daec8622ff352a8d55744140f13bcdc5.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='table1', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/8a0ec66d-a62f-b9fa-51db-a1c5c34d17d3/f4a1b344-4599-9a20-81eb-9d7a968f2df2 2024-11-08T19:22:20,057 DEBUG [RegionOpenAndInit-table1-pool-0 {}] regionserver.HRegion(898): Instantiated table1,,1731093739981.daec8622ff352a8d55744140f13bcdc5.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-08T19:22:20,057 DEBUG [RegionOpenAndInit-table1-pool-0 {}] regionserver.HRegion(1722): Closing daec8622ff352a8d55744140f13bcdc5, disabling compactions & flushes 2024-11-08T19:22:20,057 INFO [RegionOpenAndInit-table1-pool-0 {}] regionserver.HRegion(1755): Closing region table1,,1731093739981.daec8622ff352a8d55744140f13bcdc5. 2024-11-08T19:22:20,058 DEBUG [RegionOpenAndInit-table1-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on table1,,1731093739981.daec8622ff352a8d55744140f13bcdc5. 2024-11-08T19:22:20,058 DEBUG [RegionOpenAndInit-table1-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on table1,,1731093739981.daec8622ff352a8d55744140f13bcdc5. after waiting 0 ms 2024-11-08T19:22:20,058 DEBUG [RegionOpenAndInit-table1-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region table1,,1731093739981.daec8622ff352a8d55744140f13bcdc5. 2024-11-08T19:22:20,058 INFO [RegionOpenAndInit-table1-pool-0 {}] regionserver.HRegion(1973): Closed table1,,1731093739981.daec8622ff352a8d55744140f13bcdc5. 2024-11-08T19:22:20,058 DEBUG [RegionOpenAndInit-table1-pool-0 {}] regionserver.HRegion(1676): Region close journal for daec8622ff352a8d55744140f13bcdc5: Waiting for close lock at 1731093740057Disabling compacts and flushes for region at 1731093740057Disabling writes for close at 1731093740058 (+1 ms)Writing region close event to WAL at 1731093740058Closed at 1731093740058 2024-11-08T19:22:20,060 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=table1 execute state=CREATE_TABLE_ADD_TO_META 2024-11-08T19:22:20,065 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"table1,,1731093739981.daec8622ff352a8d55744140f13bcdc5.","families":{"info":[{"qualifier":"regioninfo","vlen":40,"tag":[],"timestamp":"1731093740060"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731093740060"}]},"ts":"1731093740060"} 2024-11-08T19:22:20,070 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-08T19:22:20,072 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=table1 execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-08T19:22:20,075 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"table1","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731093740072"}]},"ts":"1731093740072"} 2024-11-08T19:22:20,080 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=table1, state=ENABLING in hbase:meta 2024-11-08T19:22:20,082 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=table1, region=daec8622ff352a8d55744140f13bcdc5, ASSIGN}] 2024-11-08T19:22:20,084 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=table1, region=daec8622ff352a8d55744140f13bcdc5, ASSIGN 2024-11-08T19:22:20,086 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=table1, region=daec8622ff352a8d55744140f13bcdc5, ASSIGN; state=OFFLINE, location=692b295ad45c,37371,1731093737227; forceNewPlan=false, retain=false 2024-11-08T19:22:20,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38529 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-08T19:22:20,238 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=daec8622ff352a8d55744140f13bcdc5, regionState=OPENING, regionLocation=692b295ad45c,37371,1731093737227 2024-11-08T19:22:20,246 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=table1, region=daec8622ff352a8d55744140f13bcdc5, ASSIGN because future has completed 2024-11-08T19:22:20,247 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure daec8622ff352a8d55744140f13bcdc5, server=692b295ad45c,37371,1731093737227}] 2024-11-08T19:22:20,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38529 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-08T19:22:20,404 INFO [RS_OPEN_REGION-regionserver/692b295ad45c:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(132): Open table1,,1731093739981.daec8622ff352a8d55744140f13bcdc5. 2024-11-08T19:22:20,404 DEBUG [RS_OPEN_REGION-regionserver/692b295ad45c:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7752): Opening region: {ENCODED => daec8622ff352a8d55744140f13bcdc5, NAME => 'table1,,1731093739981.daec8622ff352a8d55744140f13bcdc5.', STARTKEY => '', ENDKEY => ''} 2024-11-08T19:22:20,405 DEBUG [RS_OPEN_REGION-regionserver/692b295ad45c:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table table1 daec8622ff352a8d55744140f13bcdc5 2024-11-08T19:22:20,405 DEBUG [RS_OPEN_REGION-regionserver/692b295ad45c:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(898): Instantiated table1,,1731093739981.daec8622ff352a8d55744140f13bcdc5.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-08T19:22:20,405 DEBUG [RS_OPEN_REGION-regionserver/692b295ad45c:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7794): checking encryption for daec8622ff352a8d55744140f13bcdc5 2024-11-08T19:22:20,405 DEBUG [RS_OPEN_REGION-regionserver/692b295ad45c:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7797): checking classloading for daec8622ff352a8d55744140f13bcdc5 2024-11-08T19:22:20,407 INFO [StoreOpener-daec8622ff352a8d55744140f13bcdc5-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=true, for column family family of region daec8622ff352a8d55744140f13bcdc5 2024-11-08T19:22:20,415 INFO [StoreOpener-daec8622ff352a8d55744140f13bcdc5-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region daec8622ff352a8d55744140f13bcdc5 columnFamilyName family 2024-11-08T19:22:20,415 DEBUG [StoreOpener-daec8622ff352a8d55744140f13bcdc5-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T19:22:20,416 INFO [StoreOpener-daec8622ff352a8d55744140f13bcdc5-1 {}] regionserver.HStore(327): Store=daec8622ff352a8d55744140f13bcdc5/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-08T19:22:20,416 DEBUG [RS_OPEN_REGION-regionserver/692b295ad45c:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1038): replaying wal for daec8622ff352a8d55744140f13bcdc5 2024-11-08T19:22:20,417 DEBUG [RS_OPEN_REGION-regionserver/692b295ad45c:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/8a0ec66d-a62f-b9fa-51db-a1c5c34d17d3/f4a1b344-4599-9a20-81eb-9d7a968f2df2/data/default/table1/daec8622ff352a8d55744140f13bcdc5 2024-11-08T19:22:20,417 DEBUG [RS_OPEN_REGION-regionserver/692b295ad45c:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/8a0ec66d-a62f-b9fa-51db-a1c5c34d17d3/f4a1b344-4599-9a20-81eb-9d7a968f2df2/data/default/table1/daec8622ff352a8d55744140f13bcdc5 2024-11-08T19:22:20,417 DEBUG [RS_OPEN_REGION-regionserver/692b295ad45c:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1048): stopping wal replay for daec8622ff352a8d55744140f13bcdc5 2024-11-08T19:22:20,417 DEBUG [RS_OPEN_REGION-regionserver/692b295ad45c:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1060): Cleaning up temporary data for daec8622ff352a8d55744140f13bcdc5 2024-11-08T19:22:20,418 DEBUG [RS_OPEN_REGION-regionserver/692b295ad45c:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1093): writing seq id for daec8622ff352a8d55744140f13bcdc5 2024-11-08T19:22:20,429 DEBUG [RS_OPEN_REGION-regionserver/692b295ad45c:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/8a0ec66d-a62f-b9fa-51db-a1c5c34d17d3/f4a1b344-4599-9a20-81eb-9d7a968f2df2/data/default/table1/daec8622ff352a8d55744140f13bcdc5/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-08T19:22:20,429 INFO [RS_OPEN_REGION-regionserver/692b295ad45c:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1114): Opened daec8622ff352a8d55744140f13bcdc5; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=70056082, jitterRate=0.0439169704914093}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-08T19:22:20,429 DEBUG [RS_OPEN_REGION-regionserver/692b295ad45c:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1122): Running coprocessor post-open hooks for daec8622ff352a8d55744140f13bcdc5 2024-11-08T19:22:20,430 DEBUG [RS_OPEN_REGION-regionserver/692b295ad45c:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1006): Region open journal for daec8622ff352a8d55744140f13bcdc5: Running coprocessor pre-open hook at 1731093740406Writing region info on filesystem at 1731093740406Initializing all the Stores at 1731093740406Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731093740406Cleaning up temporary data from old regions at 1731093740417 (+11 ms)Running coprocessor post-open hooks at 1731093740429 (+12 ms)Region opened successfully at 1731093740430 (+1 ms) 2024-11-08T19:22:20,431 INFO [RS_OPEN_REGION-regionserver/692b295ad45c:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2236): Post open deploy tasks for table1,,1731093739981.daec8622ff352a8d55744140f13bcdc5., pid=6, masterSystemTime=1731093740401 2024-11-08T19:22:20,435 DEBUG [RS_OPEN_REGION-regionserver/692b295ad45c:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2266): Finished post open deploy task for table1,,1731093739981.daec8622ff352a8d55744140f13bcdc5. 2024-11-08T19:22:20,435 INFO [RS_OPEN_REGION-regionserver/692b295ad45c:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(153): Opened table1,,1731093739981.daec8622ff352a8d55744140f13bcdc5. 2024-11-08T19:22:20,436 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=daec8622ff352a8d55744140f13bcdc5, regionState=OPEN, openSeqNum=2, regionLocation=692b295ad45c,37371,1731093737227 2024-11-08T19:22:20,439 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure daec8622ff352a8d55744140f13bcdc5, server=692b295ad45c,37371,1731093737227 because future has completed 2024-11-08T19:22:20,443 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=6, resume processing ppid=5 2024-11-08T19:22:20,444 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure daec8622ff352a8d55744140f13bcdc5, server=692b295ad45c,37371,1731093737227 in 194 msec 2024-11-08T19:22:20,446 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=5, resume processing ppid=4 2024-11-08T19:22:20,446 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=table1, region=daec8622ff352a8d55744140f13bcdc5, ASSIGN in 361 msec 2024-11-08T19:22:20,447 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=table1 execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-08T19:22:20,447 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"table1","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731093740447"}]},"ts":"1731093740447"} 2024-11-08T19:22:20,451 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=table1, state=ENABLED in hbase:meta 2024-11-08T19:22:20,452 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=table1 execute state=CREATE_TABLE_POST_OPERATION 2024-11-08T19:22:20,455 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=table1 in 456 msec 2024-11-08T19:22:20,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38529 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-08T19:22:20,645 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:table1 completed 2024-11-08T19:22:20,645 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table table1 get assigned. Timeout = 60000ms 2024-11-08T19:22:20,648 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-08T19:22:20,655 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table table1 assigned to meta. Checking AM states. 2024-11-08T19:22:20,655 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-08T19:22:20,656 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table table1 assigned. 2024-11-08T19:22:20,667 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'table1', row='row1', locateType=CURRENT is [region=table1,,1731093739981.daec8622ff352a8d55744140f13bcdc5., hostname=692b295ad45c,37371,1731093737227, seqNum=2] 2024-11-08T19:22:20,681 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing daec8622ff352a8d55744140f13bcdc5 1/1 column families, dataSize=78 B heapSize=480 B 2024-11-08T19:22:20,708 DEBUG [Time-limited test {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 9acd5a9d973f45e39a7c481bdd0554d9 2024-11-08T19:22:20,750 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/8a0ec66d-a62f-b9fa-51db-a1c5c34d17d3/f4a1b344-4599-9a20-81eb-9d7a968f2df2/data/default/table1/daec8622ff352a8d55744140f13bcdc5/.tmp/family/9acd5a9d973f45e39a7c481bdd0554d9 is 43, key is row1/family:qf1/1/Put/seqid=0 2024-11-08T19:22:20,755 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=78 B at sequenceid=6 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/8a0ec66d-a62f-b9fa-51db-a1c5c34d17d3/f4a1b344-4599-9a20-81eb-9d7a968f2df2/data/default/table1/daec8622ff352a8d55744140f13bcdc5/.tmp/family/9acd5a9d973f45e39a7c481bdd0554d9 2024-11-08T19:22:20,793 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/8a0ec66d-a62f-b9fa-51db-a1c5c34d17d3/f4a1b344-4599-9a20-81eb-9d7a968f2df2/data/default/table1/daec8622ff352a8d55744140f13bcdc5/.tmp/family/9acd5a9d973f45e39a7c481bdd0554d9 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/8a0ec66d-a62f-b9fa-51db-a1c5c34d17d3/f4a1b344-4599-9a20-81eb-9d7a968f2df2/data/default/table1/daec8622ff352a8d55744140f13bcdc5/family/9acd5a9d973f45e39a7c481bdd0554d9 2024-11-08T19:22:20,795 DEBUG [Time-limited test {}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/8a0ec66d-a62f-b9fa-51db-a1c5c34d17d3/f4a1b344-4599-9a20-81eb-9d7a968f2df2/data/default/table1/daec8622ff352a8d55744140f13bcdc5/family/9acd5a9d973f45e39a7c481bdd0554d9, delay=925 ms 2024-11-08T19:22:20,798 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/8a0ec66d-a62f-b9fa-51db-a1c5c34d17d3/f4a1b344-4599-9a20-81eb-9d7a968f2df2/data/default/table1/daec8622ff352a8d55744140f13bcdc5/family/9acd5a9d973f45e39a7c481bdd0554d9, entries=2, sequenceid=6, filesize=4.9 K 2024-11-08T19:22:20,806 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~78 B/78, heapSize ~464 B/464, currentSize=0 B/0 for daec8622ff352a8d55744140f13bcdc5 in 123ms, sequenceid=6, compaction requested=false 2024-11-08T19:22:20,807 DEBUG [Time-limited test {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'table1' 2024-11-08T19:22:20,810 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for daec8622ff352a8d55744140f13bcdc5: 2024-11-08T19:22:21,722 DEBUG [hfile-prefetch-1731093740795 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 9acd5a9d973f45e39a7c481bdd0554d9_0. 2024-11-08T19:22:21,726 DEBUG [hfile-prefetch-1731093740795 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 9acd5a9d973f45e39a7c481bdd0554d9_117. 2024-11-08T19:22:21,727 DEBUG [hfile-prefetch-1731093740795 {}] bucket.BucketCache(2282): Notifying caching completed for file file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/8a0ec66d-a62f-b9fa-51db-a1c5c34d17d3/f4a1b344-4599-9a20-81eb-9d7a968f2df2/data/default/table1/daec8622ff352a8d55744140f13bcdc5/family/9acd5a9d973f45e39a7c481bdd0554d9, with total blocks 2, and data blocks 1 2024-11-08T19:22:21,727 DEBUG [hfile-prefetch-1731093740795 {}] bucket.BucketCache(2286): iterating over 0 entries in the backing map 2024-11-08T19:22:21,728 DEBUG [hfile-prefetch-1731093740795 {}] bucket.BucketCache(2309): Prefetch executor completed for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/8a0ec66d-a62f-b9fa-51db-a1c5c34d17d3/f4a1b344-4599-9a20-81eb-9d7a968f2df2/data/default/table1/daec8622ff352a8d55744140f13bcdc5/family/9acd5a9d973f45e39a7c481bdd0554d9, but only 0 data blocks were cached. Total data blocks for file: 1. Checking for blocks pending cache in cache writer queue. 2024-11-08T19:22:21,728 DEBUG [hfile-prefetch-1731093740795 {}] bucket.BucketCache(2319): There are still blocks pending caching for file file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/8a0ec66d-a62f-b9fa-51db-a1c5c34d17d3/f4a1b344-4599-9a20-81eb-9d7a968f2df2/data/default/table1/daec8622ff352a8d55744140f13bcdc5/family/9acd5a9d973f45e39a7c481bdd0554d9. Will sleep 100ms and try the verification again. 2024-11-08T19:22:21,828 DEBUG [hfile-prefetch-1731093740795 {}] bucket.BucketCache(2282): Notifying caching completed for file file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/8a0ec66d-a62f-b9fa-51db-a1c5c34d17d3/f4a1b344-4599-9a20-81eb-9d7a968f2df2/data/default/table1/daec8622ff352a8d55744140f13bcdc5/family/9acd5a9d973f45e39a7c481bdd0554d9, with total blocks 2, and data blocks 1 2024-11-08T19:22:21,828 DEBUG [hfile-prefetch-1731093740795 {}] bucket.BucketCache(2286): iterating over 1 entries in the backing map 2024-11-08T19:22:21,828 DEBUG [hfile-prefetch-1731093740795 {}] bucket.BucketCache(2294): found block for file 9acd5a9d973f45e39a7c481bdd0554d9 in the backing map. Acquiring read lock for offset 0 2024-11-08T19:22:21,828 DEBUG [hfile-prefetch-1731093740795 {}] bucket.BucketCache(2306): File file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/8a0ec66d-a62f-b9fa-51db-a1c5c34d17d3/f4a1b344-4599-9a20-81eb-9d7a968f2df2/data/default/table1/daec8622ff352a8d55744140f13bcdc5/family/9acd5a9d973f45e39a7c481bdd0554d9 has now been fully cached. 2024-11-08T19:22:21,829 DEBUG [hfile-prefetch-1731093740795 {}] hfile.PrefetchExecutor(121): Prefetch completed for 9acd5a9d973f45e39a7c481bdd0554d9 2024-11-08T19:22:22,315 DEBUG [bucket-cache-persister {}] bucket.BucketCachePersister(47): Cache is inconsistent, persisting to disk 2024-11-08T19:22:22,316 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1488): Thread bucket-cache-persister started persisting bucket cache to file 2024-11-08T19:22:22,316 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1495): Persist in new chunked persistence format. 2024-11-08T19:22:22,316 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1674): persistToFile: before persisting backing map size: 1, fullycachedFiles size: 1, chunkSize: 10000 2024-11-08T19:22:22,336 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1681): persistToFile: after persisting backing map size: 1, fullycachedFiles size: 1 2024-11-08T19:22:22,336 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1499): PersistToFile: after persisting backing map size: 1, fullycachedFiles size: 1, file name: bucket.persistence1731093742316 2024-11-08T19:22:22,336 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1510): Thread bucket-cache-persister finished persisting bucket cache to file, renaming 2024-11-08T19:22:22,813 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-08T19:22:22,816 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48160, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-08T19:22:22,818 INFO [Time-limited test {}] hbase.SingleProcessHBaseCluster(504): Stopping Thread[RS:0;692b295ad45c:37371,5,FailOnTimeoutGroup] 2024-11-08T19:22:22,818 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '692b295ad45c,37371,1731093737227' ***** 2024-11-08T19:22:22,818 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Stopping rs 0 2024-11-08T19:22:22,818 INFO [Time-limited test {}] hfile.TestPrefetchRSClose(129): Stopped Region Server 0. 2024-11-08T19:22:22,818 INFO [RS:0;692b295ad45c:37371 {}] bucket.BucketCache(1739): Disabling cache 2024-11-08T19:22:22,819 INFO [RS:0;692b295ad45c:37371 {}] bucket.BucketCache(1763): Shutdown bucket cache: IO persistent=true; path to write=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/8a0ec66d-a62f-b9fa-51db-a1c5c34d17d3/bucket.persistence 2024-11-08T19:22:22,819 INFO [Time-limited test-BucketCacheWriter-0 {}] bucket.BucketCache$WriterThread(1272): Time-limited test-BucketCacheWriter-0 exiting, cacheEnabled=false 2024-11-08T19:22:22,819 INFO [Time-limited test-BucketCacheWriter-1 {}] bucket.BucketCache$WriterThread(1272): Time-limited test-BucketCacheWriter-1 exiting, cacheEnabled=false 2024-11-08T19:22:22,819 INFO [Time-limited test-BucketCacheWriter-2 {}] bucket.BucketCache$WriterThread(1272): Time-limited test-BucketCacheWriter-2 exiting, cacheEnabled=false 2024-11-08T19:22:22,819 INFO [RS:0;692b295ad45c:37371 {}] bucket.BucketCache(1769): Shutting down cache persister thread. 2024-11-08T19:22:22,819 WARN [bucket-cache-persister {}] bucket.BucketCachePersister(64): Interrupting BucketCachePersister thread. java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCachePersister.run(BucketCachePersister.java:45) ~[classes/:?] 2024-11-08T19:22:22,829 DEBUG [RS:0;692b295ad45c:37371 {}] bucket.BucketCache(1488): Thread RS:0;692b295ad45c:37371 started persisting bucket cache to file 2024-11-08T19:22:22,829 DEBUG [RS:0;692b295ad45c:37371 {}] bucket.BucketCache(1495): Persist in new chunked persistence format. 2024-11-08T19:22:22,829 DEBUG [RS:0;692b295ad45c:37371 {}] bucket.BucketCache(1674): persistToFile: before persisting backing map size: 1, fullycachedFiles size: 1, chunkSize: 10000 2024-11-08T19:22:22,833 DEBUG [RS:0;692b295ad45c:37371 {}] bucket.BucketCache(1681): persistToFile: after persisting backing map size: 1, fullycachedFiles size: 1 2024-11-08T19:22:22,833 DEBUG [RS:0;692b295ad45c:37371 {}] bucket.BucketCache(1499): PersistToFile: after persisting backing map size: 1, fullycachedFiles size: 1, file name: bucket.persistence1731093742829 2024-11-08T19:22:22,833 DEBUG [RS:0;692b295ad45c:37371 {}] bucket.BucketCache(1510): Thread RS:0;692b295ad45c:37371 finished persisting bucket cache to file, renaming 2024-11-08T19:22:22,833 INFO [RS:0;692b295ad45c:37371 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-08T19:22:22,833 INFO [RS:0;692b295ad45c:37371 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-08T19:22:22,833 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-08T19:22:22,834 INFO [RS:0;692b295ad45c:37371 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-08T19:22:22,834 INFO [RS:0;692b295ad45c:37371 {}] regionserver.HRegionServer(3091): Received CLOSE for daec8622ff352a8d55744140f13bcdc5 2024-11-08T19:22:22,834 INFO [RS:0;692b295ad45c:37371 {}] regionserver.HRegionServer(959): stopping server 692b295ad45c,37371,1731093737227 2024-11-08T19:22:22,834 INFO [RS:0;692b295ad45c:37371 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-08T19:22:22,835 INFO [RS:0;692b295ad45c:37371 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;692b295ad45c:37371. 2024-11-08T19:22:22,835 INFO [RS_CLOSE_REGION-regionserver/692b295ad45c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1662): Closing region table1,,1731093739981.daec8622ff352a8d55744140f13bcdc5. during a graceful stop, and cache persistence is on, so setting evict on close to false. 2024-11-08T19:22:22,835 DEBUG [RS:0;692b295ad45c:37371 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-08T19:22:22,835 DEBUG [RS_CLOSE_REGION-regionserver/692b295ad45c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing daec8622ff352a8d55744140f13bcdc5, disabling compactions & flushes 2024-11-08T19:22:22,835 INFO [RS_CLOSE_REGION-regionserver/692b295ad45c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region table1,,1731093739981.daec8622ff352a8d55744140f13bcdc5. 2024-11-08T19:22:22,835 DEBUG [RS_CLOSE_REGION-regionserver/692b295ad45c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on table1,,1731093739981.daec8622ff352a8d55744140f13bcdc5. 2024-11-08T19:22:22,835 DEBUG [RS_CLOSE_REGION-regionserver/692b295ad45c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on table1,,1731093739981.daec8622ff352a8d55744140f13bcdc5. after waiting 0 ms 2024-11-08T19:22:22,835 DEBUG [RS_CLOSE_REGION-regionserver/692b295ad45c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region table1,,1731093739981.daec8622ff352a8d55744140f13bcdc5. 2024-11-08T19:22:22,841 DEBUG [RS:0;692b295ad45c:37371 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T19:22:22,841 DEBUG [RS_CLOSE_REGION-regionserver/692b295ad45c:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/8a0ec66d-a62f-b9fa-51db-a1c5c34d17d3/f4a1b344-4599-9a20-81eb-9d7a968f2df2/data/default/table1/daec8622ff352a8d55744140f13bcdc5/recovered.edits/9.seqid, newMaxSeqId=9, maxSeqId=1 2024-11-08T19:22:22,841 INFO [RS:0;692b295ad45c:37371 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-08T19:22:22,841 INFO [RS:0;692b295ad45c:37371 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-08T19:22:22,841 INFO [RS:0;692b295ad45c:37371 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-08T19:22:22,842 INFO [RS:0;692b295ad45c:37371 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-11-08T19:22:22,842 INFO [RS_CLOSE_REGION-regionserver/692b295ad45c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed table1,,1731093739981.daec8622ff352a8d55744140f13bcdc5. 2024-11-08T19:22:22,842 DEBUG [RS_CLOSE_REGION-regionserver/692b295ad45c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for daec8622ff352a8d55744140f13bcdc5: Waiting for close lock at 1731093742834Running coprocessor pre-close hooks at 1731093742835 (+1 ms)Disabling compacts and flushes for region at 1731093742835Disabling writes for close at 1731093742835Writing region close event to WAL at 1731093742837 (+2 ms)Running coprocessor post-close hooks at 1731093742841 (+4 ms)Closed at 1731093742842 (+1 ms) 2024-11-08T19:22:22,843 INFO [RS:0;692b295ad45c:37371 {}] regionserver.HRegionServer(1321): Waiting on 1 regions to close 2024-11-08T19:22:22,843 INFO [RS_CLOSE_META-regionserver/692b295ad45c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1662): Closing region hbase:meta,,1.1588230740 during a graceful stop, and cache persistence is on, so setting evict on close to false. 2024-11-08T19:22:22,843 DEBUG [RS:0;692b295ad45c:37371 {}] regionserver.HRegionServer(1325): Online Regions={1588230740=hbase:meta,,1.1588230740} 2024-11-08T19:22:22,843 DEBUG [RS_CLOSE_META-regionserver/692b295ad45c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-08T19:22:22,843 INFO [RS_CLOSE_META-regionserver/692b295ad45c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-08T19:22:22,843 DEBUG [RS_CLOSE_REGION-regionserver/692b295ad45c:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed table1,,1731093739981.daec8622ff352a8d55744140f13bcdc5. 2024-11-08T19:22:22,843 DEBUG [RS_CLOSE_META-regionserver/692b295ad45c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-08T19:22:22,843 DEBUG [RS_CLOSE_META-regionserver/692b295ad45c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-08T19:22:22,843 DEBUG [RS_CLOSE_META-regionserver/692b295ad45c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-08T19:22:22,843 DEBUG [RS:0;692b295ad45c:37371 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-08T19:22:22,843 INFO [RS_CLOSE_META-regionserver/692b295ad45c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=1.20 KB heapSize=3.23 KB 2024-11-08T19:22:22,848 INFO [regionserver/692b295ad45c:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-08T19:22:22,854 DEBUG [RS_CLOSE_META-regionserver/692b295ad45c:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 3cb8a7213df04729924f2dcf47d9efa8 2024-11-08T19:22:22,870 DEBUG [RS_CLOSE_META-regionserver/692b295ad45c:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/8a0ec66d-a62f-b9fa-51db-a1c5c34d17d3/f4a1b344-4599-9a20-81eb-9d7a968f2df2/data/hbase/meta/1588230740/.tmp/info/3cb8a7213df04729924f2dcf47d9efa8 is 133, key is table1,,1731093739981.daec8622ff352a8d55744140f13bcdc5./info:regioninfo/1731093740435/Put/seqid=0 2024-11-08T19:22:22,871 INFO [RS_CLOSE_META-regionserver/692b295ad45c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.05 KB at sequenceid=11 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/8a0ec66d-a62f-b9fa-51db-a1c5c34d17d3/f4a1b344-4599-9a20-81eb-9d7a968f2df2/data/hbase/meta/1588230740/.tmp/info/3cb8a7213df04729924f2dcf47d9efa8 2024-11-08T19:22:22,880 DEBUG [RS_CLOSE_META-regionserver/692b295ad45c:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFile$WriterFactory(308): Unable to set drop behind on d61a8158ec7a4f22a07dc9db6139f86d 2024-11-08T19:22:22,892 INFO [regionserver/692b295ad45c:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: MemstoreFlusherChore was stopped 2024-11-08T19:22:22,892 INFO [regionserver/692b295ad45c:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: CompactionChecker was stopped 2024-11-08T19:22:22,894 DEBUG [RS_CLOSE_META-regionserver/692b295ad45c:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/8a0ec66d-a62f-b9fa-51db-a1c5c34d17d3/f4a1b344-4599-9a20-81eb-9d7a968f2df2/data/hbase/meta/1588230740/.tmp/ns/d61a8158ec7a4f22a07dc9db6139f86d is 43, key is default/ns:d/1731093739743/Put/seqid=0 2024-11-08T19:22:22,895 INFO [RS_CLOSE_META-regionserver/692b295ad45c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=74 B at sequenceid=11 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/8a0ec66d-a62f-b9fa-51db-a1c5c34d17d3/f4a1b344-4599-9a20-81eb-9d7a968f2df2/data/hbase/meta/1588230740/.tmp/ns/d61a8158ec7a4f22a07dc9db6139f86d 2024-11-08T19:22:22,904 DEBUG [RS_CLOSE_META-regionserver/692b295ad45c:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFile$WriterFactory(308): Unable to set drop behind on c4d8191d287343dbbc616e48a0a67032 2024-11-08T19:22:22,919 DEBUG [RS_CLOSE_META-regionserver/692b295ad45c:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/8a0ec66d-a62f-b9fa-51db-a1c5c34d17d3/f4a1b344-4599-9a20-81eb-9d7a968f2df2/data/hbase/meta/1588230740/.tmp/table/c4d8191d287343dbbc616e48a0a67032 is 42, key is table1/table:state/1731093740447/Put/seqid=0 2024-11-08T19:22:22,920 INFO [RS_CLOSE_META-regionserver/692b295ad45c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=76 B at sequenceid=11 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/8a0ec66d-a62f-b9fa-51db-a1c5c34d17d3/f4a1b344-4599-9a20-81eb-9d7a968f2df2/data/hbase/meta/1588230740/.tmp/table/c4d8191d287343dbbc616e48a0a67032 2024-11-08T19:22:22,923 DEBUG [RS_CLOSE_META-regionserver/692b295ad45c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/8a0ec66d-a62f-b9fa-51db-a1c5c34d17d3/f4a1b344-4599-9a20-81eb-9d7a968f2df2/data/hbase/meta/1588230740/.tmp/info/3cb8a7213df04729924f2dcf47d9efa8 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/8a0ec66d-a62f-b9fa-51db-a1c5c34d17d3/f4a1b344-4599-9a20-81eb-9d7a968f2df2/data/hbase/meta/1588230740/info/3cb8a7213df04729924f2dcf47d9efa8 2024-11-08T19:22:22,925 DEBUG [RS_CLOSE_META-regionserver/692b295ad45c:0-0 {event_type=M_RS_CLOSE_META}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/8a0ec66d-a62f-b9fa-51db-a1c5c34d17d3/f4a1b344-4599-9a20-81eb-9d7a968f2df2/data/hbase/meta/1588230740/info/3cb8a7213df04729924f2dcf47d9efa8, delay=907 ms 2024-11-08T19:22:22,927 INFO [RS_CLOSE_META-regionserver/692b295ad45c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/8a0ec66d-a62f-b9fa-51db-a1c5c34d17d3/f4a1b344-4599-9a20-81eb-9d7a968f2df2/data/hbase/meta/1588230740/info/3cb8a7213df04729924f2dcf47d9efa8, entries=10, sequenceid=11, filesize=6.3 K 2024-11-08T19:22:22,927 DEBUG [RS_CLOSE_META-regionserver/692b295ad45c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/8a0ec66d-a62f-b9fa-51db-a1c5c34d17d3/f4a1b344-4599-9a20-81eb-9d7a968f2df2/data/hbase/meta/1588230740/.tmp/ns/d61a8158ec7a4f22a07dc9db6139f86d as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/8a0ec66d-a62f-b9fa-51db-a1c5c34d17d3/f4a1b344-4599-9a20-81eb-9d7a968f2df2/data/hbase/meta/1588230740/ns/d61a8158ec7a4f22a07dc9db6139f86d 2024-11-08T19:22:22,929 DEBUG [RS_CLOSE_META-regionserver/692b295ad45c:0-0 {event_type=M_RS_CLOSE_META}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/8a0ec66d-a62f-b9fa-51db-a1c5c34d17d3/f4a1b344-4599-9a20-81eb-9d7a968f2df2/data/hbase/meta/1588230740/ns/d61a8158ec7a4f22a07dc9db6139f86d, delay=968 ms 2024-11-08T19:22:22,931 INFO [RS_CLOSE_META-regionserver/692b295ad45c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/8a0ec66d-a62f-b9fa-51db-a1c5c34d17d3/f4a1b344-4599-9a20-81eb-9d7a968f2df2/data/hbase/meta/1588230740/ns/d61a8158ec7a4f22a07dc9db6139f86d, entries=2, sequenceid=11, filesize=5.0 K 2024-11-08T19:22:22,932 DEBUG [RS_CLOSE_META-regionserver/692b295ad45c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/8a0ec66d-a62f-b9fa-51db-a1c5c34d17d3/f4a1b344-4599-9a20-81eb-9d7a968f2df2/data/hbase/meta/1588230740/.tmp/table/c4d8191d287343dbbc616e48a0a67032 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/8a0ec66d-a62f-b9fa-51db-a1c5c34d17d3/f4a1b344-4599-9a20-81eb-9d7a968f2df2/data/hbase/meta/1588230740/table/c4d8191d287343dbbc616e48a0a67032 2024-11-08T19:22:22,933 DEBUG [RS_CLOSE_META-regionserver/692b295ad45c:0-0 {event_type=M_RS_CLOSE_META}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/8a0ec66d-a62f-b9fa-51db-a1c5c34d17d3/f4a1b344-4599-9a20-81eb-9d7a968f2df2/data/hbase/meta/1588230740/table/c4d8191d287343dbbc616e48a0a67032, delay=958 ms 2024-11-08T19:22:22,935 INFO [RS_CLOSE_META-regionserver/692b295ad45c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/8a0ec66d-a62f-b9fa-51db-a1c5c34d17d3/f4a1b344-4599-9a20-81eb-9d7a968f2df2/data/hbase/meta/1588230740/table/c4d8191d287343dbbc616e48a0a67032, entries=2, sequenceid=11, filesize=5.1 K 2024-11-08T19:22:22,936 INFO [RS_CLOSE_META-regionserver/692b295ad45c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~1.20 KB/1227, heapSize ~2.94 KB/3008, currentSize=0 B/0 for 1588230740 in 93ms, sequenceid=11, compaction requested=false 2024-11-08T19:22:22,936 DEBUG [RS_CLOSE_META-regionserver/692b295ad45c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta' 2024-11-08T19:22:22,938 DEBUG [StoreFileCloser-1588230740-info-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/8a0ec66d-a62f-b9fa-51db-a1c5c34d17d3/f4a1b344-4599-9a20-81eb-9d7a968f2df2/data/hbase/meta/1588230740/info/3cb8a7213df04729924f2dcf47d9efa8 2024-11-08T19:22:22,939 DEBUG [StoreFileCloser-1588230740-ns-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/8a0ec66d-a62f-b9fa-51db-a1c5c34d17d3/f4a1b344-4599-9a20-81eb-9d7a968f2df2/data/hbase/meta/1588230740/ns/d61a8158ec7a4f22a07dc9db6139f86d 2024-11-08T19:22:22,940 DEBUG [StoreFileCloser-1588230740-table-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/8a0ec66d-a62f-b9fa-51db-a1c5c34d17d3/f4a1b344-4599-9a20-81eb-9d7a968f2df2/data/hbase/meta/1588230740/table/c4d8191d287343dbbc616e48a0a67032 2024-11-08T19:22:22,944 DEBUG [RS_CLOSE_META-regionserver/692b295ad45c:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/8a0ec66d-a62f-b9fa-51db-a1c5c34d17d3/f4a1b344-4599-9a20-81eb-9d7a968f2df2/data/hbase/meta/1588230740/recovered.edits/14.seqid, newMaxSeqId=14, maxSeqId=1 2024-11-08T19:22:22,945 DEBUG [RS_CLOSE_META-regionserver/692b295ad45c:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-08T19:22:22,945 INFO [RS_CLOSE_META-regionserver/692b295ad45c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-08T19:22:22,945 DEBUG [RS_CLOSE_META-regionserver/692b295ad45c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1731093742843Running coprocessor pre-close hooks at 1731093742843Disabling compacts and flushes for region at 1731093742843Disabling writes for close at 1731093742843Obtaining lock to block concurrent updates at 1731093742843Preparing flush snapshotting stores in 1588230740 at 1731093742843Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=1227, getHeapSize=3248, getOffHeapSize=0, getCellsCount=14 at 1731093742844 (+1 ms)Flushing stores of hbase:meta,,1.1588230740 at 1731093742844Flushing 1588230740/info: creating writer at 1731093742845 (+1 ms)Flushing 1588230740/info: appending metadata at 1731093742869 (+24 ms)Flushing 1588230740/info: closing flushed file at 1731093742869Flushing 1588230740/ns: creating writer at 1731093742874 (+5 ms)Flushing 1588230740/ns: appending metadata at 1731093742893 (+19 ms)Flushing 1588230740/ns: closing flushed file at 1731093742893Flushing 1588230740/table: creating writer at 1731093742898 (+5 ms)Flushing 1588230740/table: appending metadata at 1731093742919 (+21 ms)Flushing 1588230740/table: closing flushed file at 1731093742919Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@51470976: reopening flushed file at 1731093742923 (+4 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@660b385f: reopening flushed file at 1731093742927 (+4 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@66b9350c: reopening flushed file at 1731093742931 (+4 ms)Finished flush of dataSize ~1.20 KB/1227, heapSize ~2.94 KB/3008, currentSize=0 B/0 for 1588230740 in 93ms, sequenceid=11, compaction requested=false at 1731093742936 (+5 ms)Writing region close event to WAL at 1731093742940 (+4 ms)Running coprocessor post-close hooks at 1731093742944 (+4 ms)Closed at 1731093742945 (+1 ms) 2024-11-08T19:22:22,945 DEBUG [RS_CLOSE_META-regionserver/692b295ad45c:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-11-08T19:22:23,044 INFO [RS:0;692b295ad45c:37371 {}] regionserver.HRegionServer(976): stopping server 692b295ad45c,37371,1731093737227; all regions closed. 2024-11-08T19:22:23,051 DEBUG [RS:0;692b295ad45c:37371 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/8a0ec66d-a62f-b9fa-51db-a1c5c34d17d3/f4a1b344-4599-9a20-81eb-9d7a968f2df2/oldWALs 2024-11-08T19:22:23,052 INFO [RS:0;692b295ad45c:37371 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 692b295ad45c%2C37371%2C1731093737227.meta:.meta(num 1731093739446) 2024-11-08T19:22:23,055 DEBUG [RS:0;692b295ad45c:37371 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/8a0ec66d-a62f-b9fa-51db-a1c5c34d17d3/f4a1b344-4599-9a20-81eb-9d7a968f2df2/oldWALs 2024-11-08T19:22:23,055 INFO [RS:0;692b295ad45c:37371 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 692b295ad45c%2C37371%2C1731093737227:(num 1731093739020) 2024-11-08T19:22:23,055 DEBUG [RS:0;692b295ad45c:37371 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T19:22:23,055 INFO [RS:0;692b295ad45c:37371 {}] regionserver.LeaseManager(133): Closed leases 2024-11-08T19:22:23,055 INFO [RS:0;692b295ad45c:37371 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-08T19:22:23,056 INFO [RS:0;692b295ad45c:37371 {}] hbase.ChoreService(370): Chore service for: regionserver/692b295ad45c:0 had [ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-11-08T19:22:23,056 INFO [RS:0;692b295ad45c:37371 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-08T19:22:23,056 INFO [regionserver/692b295ad45c:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-08T19:22:23,056 INFO [RS:0;692b295ad45c:37371 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:37371 2024-11-08T19:22:23,158 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37371-0x1011be626410001, quorum=127.0.0.1:63355, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/692b295ad45c,37371,1731093737227 2024-11-08T19:22:23,158 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38529-0x1011be626410000, quorum=127.0.0.1:63355, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-08T19:22:23,158 INFO [RS:0;692b295ad45c:37371 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-08T19:22:23,159 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [692b295ad45c,37371,1731093737227] 2024-11-08T19:22:23,264 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/692b295ad45c,37371,1731093737227 already deleted, retry=false 2024-11-08T19:22:23,264 INFO [RegionServerTracker-0 {}] master.ServerManager(695): Processing expiration of 692b295ad45c,37371,1731093737227 on 692b295ad45c,38529,1731093736393 2024-11-08T19:22:23,268 DEBUG [RegionServerTracker-0 {}] procedure2.ProcedureExecutor(1139): Stored pid=7, state=RUNNABLE:SERVER_CRASH_START, hasLock=false; ServerCrashProcedure 692b295ad45c,37371,1731093737227, splitWal=true, meta=true 2024-11-08T19:22:23,270 INFO [RegionServerTracker-0 {}] assignment.AssignmentManager(1991): Scheduled ServerCrashProcedure pid=7 for 692b295ad45c,37371,1731093737227 (carryingMeta=true) 692b295ad45c,37371,1731093737227/CRASHED/regionCount=2/lock=java.util.concurrent.locks.ReentrantReadWriteLock@2103847a[Write locks = 1, Read locks = 0], oldState=ONLINE. 2024-11-08T19:22:23,271 INFO [PEWorker-1 {}] procedure.ServerCrashProcedure(169): Start pid=7, state=RUNNABLE:SERVER_CRASH_START, hasLock=true; ServerCrashProcedure 692b295ad45c,37371,1731093737227, splitWal=true, meta=true 2024-11-08T19:22:23,272 INFO [PEWorker-1 {}] procedure.ServerCrashProcedure(339): Splitting WALs pid=7, state=RUNNABLE:SERVER_CRASH_SPLIT_META_LOGS, hasLock=true; ServerCrashProcedure 692b295ad45c,37371,1731093737227, splitWal=true, meta=true, isMeta: true 2024-11-08T19:22:23,272 DEBUG [PEWorker-1 {}] master.MasterWalManager(329): Renamed region directory: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/8a0ec66d-a62f-b9fa-51db-a1c5c34d17d3/f4a1b344-4599-9a20-81eb-9d7a968f2df2/WALs/692b295ad45c,37371,1731093737227-splitting 2024-11-08T19:22:23,273 INFO [PEWorker-1 {}] master.SplitLogManager(171): file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/8a0ec66d-a62f-b9fa-51db-a1c5c34d17d3/f4a1b344-4599-9a20-81eb-9d7a968f2df2/WALs/692b295ad45c,37371,1731093737227-splitting dir is empty, no logs to split. 2024-11-08T19:22:23,273 INFO [PEWorker-1 {}] master.SplitWALManager(105): 692b295ad45c,37371,1731093737227 WAL count=0, meta=true 2024-11-08T19:22:23,275 INFO [PEWorker-1 {}] master.SplitLogManager(171): file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/8a0ec66d-a62f-b9fa-51db-a1c5c34d17d3/f4a1b344-4599-9a20-81eb-9d7a968f2df2/WALs/692b295ad45c,37371,1731093737227-splitting dir is empty, no logs to split. 2024-11-08T19:22:23,275 INFO [PEWorker-1 {}] master.SplitWALManager(105): 692b295ad45c,37371,1731093737227 WAL count=0, meta=true 2024-11-08T19:22:23,275 DEBUG [PEWorker-1 {}] procedure.ServerCrashProcedure(329): Check if 692b295ad45c,37371,1731093737227 WAL splitting is done? wals=0, meta=true 2024-11-08T19:22:23,277 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=8, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-11-08T19:22:23,278 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=8, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-08T19:22:23,279 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=8, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-11-08T19:22:23,344 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37371-0x1011be626410001, quorum=127.0.0.1:63355, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-08T19:22:23,344 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37371-0x1011be626410001, quorum=127.0.0.1:63355, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-08T19:22:23,345 INFO [RS:0;692b295ad45c:37371 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-08T19:22:23,345 INFO [RS:0;692b295ad45c:37371 {}] regionserver.HRegionServer(1031): Exiting; stopping=692b295ad45c,37371,1731093737227; zookeeper connection closed. 2024-11-08T19:22:23,346 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@6384752b {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@6384752b 2024-11-08T19:22:23,430 WARN [692b295ad45c:38529 {}] assignment.AssignmentManager(2443): No servers available; cannot place 1 unassigned regions. 2024-11-08T19:22:23,819 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster 2024-11-08T19:22:23,819 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-08T19:22:23,819 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79) at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611) at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.io.hfile.TestPrefetchRSClose.tearDown(TestPrefetchRSClose.java:136) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-08T19:22:23,819 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T19:22:23,820 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T19:22:23,820 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-08T19:22:23,820 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2024-11-08T19:22:23,820 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=1776925241, stopped=false 2024-11-08T19:22:23,821 INFO [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=692b295ad45c,38529,1731093736393 2024-11-08T19:22:23,821 INFO [Time-limited test {}] master.HMaster(3321): ***** STOPPING master '692b295ad45c,38529,1731093736393' ***** 2024-11-08T19:22:23,821 INFO [Time-limited test {}] master.HMaster(3323): STOPPED: OnlineServer=0 right after cluster shutdown set 2024-11-08T19:22:23,821 INFO [M:0;692b295ad45c:38529 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-08T19:22:23,821 INFO [M:0;692b295ad45c:38529 {}] client.AsyncConnectionImpl(233): Connection has been closed by M:0;692b295ad45c:38529. 2024-11-08T19:22:23,821 DEBUG [M:0;692b295ad45c:38529 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.master.HMaster.run(HMaster.java:630) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-08T19:22:23,821 DEBUG [M:0;692b295ad45c:38529 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T19:22:23,822 INFO [M:0;692b295ad45c:38529 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-08T19:22:23,822 DEBUG [M:0;692b295ad45c:38529 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2024-11-08T19:22:23,822 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2024-11-08T19:22:23,822 DEBUG [M:0;692b295ad45c:38529 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2024-11-08T19:22:23,822 DEBUG [master/692b295ad45c:0:becomeActiveMaster-HFileCleaner.large.0-1731093738626 {}] cleaner.HFileCleaner(306): Exit Thread[master/692b295ad45c:0:becomeActiveMaster-HFileCleaner.large.0-1731093738626,5,FailOnTimeoutGroup] 2024-11-08T19:22:23,822 DEBUG [master/692b295ad45c:0:becomeActiveMaster-HFileCleaner.small.0-1731093738627 {}] cleaner.HFileCleaner(306): Exit Thread[master/692b295ad45c:0:becomeActiveMaster-HFileCleaner.small.0-1731093738627,5,FailOnTimeoutGroup] 2024-11-08T19:22:23,822 INFO [M:0;692b295ad45c:38529 {}] hbase.ChoreService(370): Chore service for: master/692b295ad45c:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown 2024-11-08T19:22:23,822 INFO [M:0;692b295ad45c:38529 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-08T19:22:23,823 DEBUG [M:0;692b295ad45c:38529 {}] master.HMaster(1795): Stopping service threads 2024-11-08T19:22:23,823 INFO [M:0;692b295ad45c:38529 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2024-11-08T19:22:23,823 INFO [M:0;692b295ad45c:38529 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-08T19:22:23,825 INFO [M:0;692b295ad45c:38529 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-08T19:22:23,825 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2024-11-08T19:22:23,843 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38529-0x1011be626410000, quorum=127.0.0.1:63355, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-11-08T19:22:23,843 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38529-0x1011be626410000, quorum=127.0.0.1:63355, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-08T19:22:23,843 DEBUG [M:0;692b295ad45c:38529 {}] zookeeper.ZKUtil(347): master:38529-0x1011be626410000, quorum=127.0.0.1:63355, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error) 2024-11-08T19:22:23,843 WARN [M:0;692b295ad45c:38529 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null 2024-11-08T19:22:23,844 INFO [M:0;692b295ad45c:38529 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/8a0ec66d-a62f-b9fa-51db-a1c5c34d17d3/f4a1b344-4599-9a20-81eb-9d7a968f2df2/.lastflushedseqids 2024-11-08T19:22:23,853 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38529-0x1011be626410000, quorum=127.0.0.1:63355, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-08T19:22:23,854 INFO [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 1 regionserver(s) complete 2024-11-08T19:22:23,854 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:38529-0x1011be626410000, quorum=127.0.0.1:63355, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-08T19:22:23,857 INFO [M:0;692b295ad45c:38529 {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-08T19:22:23,930 DEBUG [692b295ad45c:38529 {}] assignment.AssignmentManager(2447): Stopped! Dropping assign of 1 queued regions. 2024-11-08T19:22:23,931 INFO [M:0;692b295ad45c:38529 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2024-11-08T19:22:23,931 DEBUG [M:0;692b295ad45c:38529 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-08T19:22:23,931 INFO [M:0;692b295ad45c:38529 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-08T19:22:23,931 DEBUG [M:0;692b295ad45c:38529 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-08T19:22:23,931 DEBUG [M:0;692b295ad45c:38529 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-08T19:22:23,931 DEBUG [M:0;692b295ad45c:38529 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-08T19:22:23,932 INFO [M:0;692b295ad45c:38529 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=24.18 KB heapSize=30.80 KB 2024-11-08T19:22:23,944 DEBUG [M:0;692b295ad45c:38529 {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on d23454bc24e7454099d220c3a14e5e13 2024-11-08T19:22:23,961 DEBUG [M:0;692b295ad45c:38529 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/8a0ec66d-a62f-b9fa-51db-a1c5c34d17d3/f4a1b344-4599-9a20-81eb-9d7a968f2df2/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/d23454bc24e7454099d220c3a14e5e13 is 82, key is hbase:meta,,1/info:regioninfo/1731093739531/Put/seqid=0 2024-11-08T19:22:23,961 INFO [M:0;692b295ad45c:38529 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=504 B at sequenceid=64 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/8a0ec66d-a62f-b9fa-51db-a1c5c34d17d3/f4a1b344-4599-9a20-81eb-9d7a968f2df2/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/d23454bc24e7454099d220c3a14e5e13 2024-11-08T19:22:23,970 DEBUG [M:0;692b295ad45c:38529 {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 4c185824f6ee43c09aede9230b342605 2024-11-08T19:22:23,991 DEBUG [M:0;692b295ad45c:38529 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/8a0ec66d-a62f-b9fa-51db-a1c5c34d17d3/f4a1b344-4599-9a20-81eb-9d7a968f2df2/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/4c185824f6ee43c09aede9230b342605 is 707, key is \x00\x00\x00\x00\x00\x00\x00\x04/proc:d/1731093740454/Put/seqid=0 2024-11-08T19:22:23,992 INFO [M:0;692b295ad45c:38529 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=23.57 KB at sequenceid=64 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/8a0ec66d-a62f-b9fa-51db-a1c5c34d17d3/f4a1b344-4599-9a20-81eb-9d7a968f2df2/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/4c185824f6ee43c09aede9230b342605 2024-11-08T19:22:24,005 DEBUG [M:0;692b295ad45c:38529 {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 81cfabb172ea427f9938f80af6968871 2024-11-08T19:22:24,018 DEBUG [M:0;692b295ad45c:38529 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/8a0ec66d-a62f-b9fa-51db-a1c5c34d17d3/f4a1b344-4599-9a20-81eb-9d7a968f2df2/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/81cfabb172ea427f9938f80af6968871 is 58, key is 692b295ad45c,37371,1731093737227/rs:/1731093743271/DeleteFamily/seqid=0 2024-11-08T19:22:24,019 INFO [M:0;692b295ad45c:38529 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=119 B at sequenceid=64 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/8a0ec66d-a62f-b9fa-51db-a1c5c34d17d3/f4a1b344-4599-9a20-81eb-9d7a968f2df2/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/81cfabb172ea427f9938f80af6968871 2024-11-08T19:22:24,021 INFO [M:0;692b295ad45c:38529 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 81cfabb172ea427f9938f80af6968871 2024-11-08T19:22:24,022 DEBUG [M:0;692b295ad45c:38529 {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/8a0ec66d-a62f-b9fa-51db-a1c5c34d17d3/f4a1b344-4599-9a20-81eb-9d7a968f2df2/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/d23454bc24e7454099d220c3a14e5e13 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/8a0ec66d-a62f-b9fa-51db-a1c5c34d17d3/f4a1b344-4599-9a20-81eb-9d7a968f2df2/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/d23454bc24e7454099d220c3a14e5e13 2024-11-08T19:22:24,024 INFO [M:0;692b295ad45c:38529 {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/8a0ec66d-a62f-b9fa-51db-a1c5c34d17d3/f4a1b344-4599-9a20-81eb-9d7a968f2df2/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/d23454bc24e7454099d220c3a14e5e13, entries=8, sequenceid=64, filesize=5.5 K 2024-11-08T19:22:24,025 DEBUG [M:0;692b295ad45c:38529 {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/8a0ec66d-a62f-b9fa-51db-a1c5c34d17d3/f4a1b344-4599-9a20-81eb-9d7a968f2df2/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/4c185824f6ee43c09aede9230b342605 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/8a0ec66d-a62f-b9fa-51db-a1c5c34d17d3/f4a1b344-4599-9a20-81eb-9d7a968f2df2/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/4c185824f6ee43c09aede9230b342605 2024-11-08T19:22:24,028 INFO [M:0;692b295ad45c:38529 {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/8a0ec66d-a62f-b9fa-51db-a1c5c34d17d3/f4a1b344-4599-9a20-81eb-9d7a968f2df2/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/4c185824f6ee43c09aede9230b342605, entries=8, sequenceid=64, filesize=6.5 K 2024-11-08T19:22:24,028 DEBUG [M:0;692b295ad45c:38529 {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/8a0ec66d-a62f-b9fa-51db-a1c5c34d17d3/f4a1b344-4599-9a20-81eb-9d7a968f2df2/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/81cfabb172ea427f9938f80af6968871 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/8a0ec66d-a62f-b9fa-51db-a1c5c34d17d3/f4a1b344-4599-9a20-81eb-9d7a968f2df2/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/81cfabb172ea427f9938f80af6968871 2024-11-08T19:22:24,031 INFO [M:0;692b295ad45c:38529 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 81cfabb172ea427f9938f80af6968871 2024-11-08T19:22:24,031 INFO [M:0;692b295ad45c:38529 {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/8a0ec66d-a62f-b9fa-51db-a1c5c34d17d3/f4a1b344-4599-9a20-81eb-9d7a968f2df2/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/81cfabb172ea427f9938f80af6968871, entries=1, sequenceid=64, filesize=5.2 K 2024-11-08T19:22:24,033 INFO [M:0;692b295ad45c:38529 {}] regionserver.HRegion(3140): Finished flush of dataSize ~24.18 KB/24759, heapSize ~30.51 KB/31240, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 101ms, sequenceid=64, compaction requested=false 2024-11-08T19:22:24,036 INFO [M:0;692b295ad45c:38529 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-08T19:22:24,036 DEBUG [M:0;692b295ad45c:38529 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1731093743931Disabling compacts and flushes for region at 1731093743931Disabling writes for close at 1731093743931Obtaining lock to block concurrent updates at 1731093743932 (+1 ms)Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1731093743932Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=24759, getHeapSize=31480, getOffHeapSize=0, getCellsCount=76 at 1731093743933 (+1 ms)Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1731093743933Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1731093743933Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1731093743960 (+27 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1731093743960Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1731093743963 (+3 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1731093743990 (+27 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1731093743990Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1731093743995 (+5 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1731093744017 (+22 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1731093744017Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@6f8c3d34: reopening flushed file at 1731093744021 (+4 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@75bf8330: reopening flushed file at 1731093744025 (+4 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@35baf769: reopening flushed file at 1731093744028 (+3 ms)Finished flush of dataSize ~24.18 KB/24759, heapSize ~30.51 KB/31240, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 101ms, sequenceid=64, compaction requested=false at 1731093744033 (+5 ms)Writing region close event to WAL at 1731093744036 (+3 ms)Closed at 1731093744036 2024-11-08T19:22:24,040 INFO [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-08T19:22:24,040 INFO [M:0;692b295ad45c:38529 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down. 2024-11-08T19:22:24,041 INFO [M:0;692b295ad45c:38529 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:38529 2024-11-08T19:22:24,041 INFO [M:0;692b295ad45c:38529 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-08T19:22:24,159 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38529-0x1011be626410000, quorum=127.0.0.1:63355, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-08T19:22:24,159 INFO [M:0;692b295ad45c:38529 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-08T19:22:24,159 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38529-0x1011be626410000, quorum=127.0.0.1:63355, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-08T19:22:24,165 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers 2024-11-08T19:22:24,199 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down 2024-11-08T19:22:24,200 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-08T19:22:24,209 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: io.hfile.TestPrefetchRSClose#testPrefetchPersistence Thread=45 (was 12) Potentially hanging thread: Async-Client-Retry-Timer-pool-0 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.waitForNextTick(HashedWheelTimer.java:598) app//org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:494) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Idle-Rpc-Conn-Sweeper-pool-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-3-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: hfile-prefetch-1731093742925 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1177) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HBase-Metrics2-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-3-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: hfile-prefetch-1731093740795 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1177) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: SessionTracker java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.zookeeper.server.SessionTrackerImpl.run(SessionTrackerImpl.java:163) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: hfile-prefetch-1731093742933 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1177) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RpcClient-timer-pool-0 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.waitForNextTick(HashedWheelTimer.java:598) app//org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:494) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HMaster-EventLoopGroup-1-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: hfile-prefetch-1731093742929 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1170) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: weak-ref-cleaner-strictcontextstorage java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:155) java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:176) app//io.opentelemetry.context.StrictContextStorage$PendingScopes.run(StrictContextStorage.java:269) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: region-location-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1170) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: master/692b295ad45c:0:becomeActiveMaster-MemStoreChunkPool Statistics java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Monitor thread for TaskMonitor java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hbase.monitoring.TaskMonitor$MonitorRunnable.run(TaskMonitor.java:325) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HMaster-EventLoopGroup-1-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ForkJoinPool.commonPool-worker-3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.ForkJoinPool.awaitWork(ForkJoinPool.java:1724) java.base@17.0.11/java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1623) java.base@17.0.11/java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:165) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-3-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Timer for 'HBase' metrics system java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.util.TimerThread.mainLoop(Timer.java:563) java.base@17.0.11/java.util.TimerThread.run(Timer.java:516) Potentially hanging thread: ForkJoinPool.commonPool-worker-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkUntil(LockSupport.java:410) java.base@17.0.11/java.util.concurrent.ForkJoinPool.awaitWork(ForkJoinPool.java:1726) java.base@17.0.11/java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1623) java.base@17.0.11/java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:165) Potentially hanging thread: Time-limited test.named-queue-events-pool-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) app//com.lmax.disruptor.BlockingWaitStrategy.waitFor(BlockingWaitStrategy.java:47) app//com.lmax.disruptor.ProcessingSequenceBarrier.waitFor(ProcessingSequenceBarrier.java:56) app//com.lmax.disruptor.BatchEventProcessor.processEvents(BatchEventProcessor.java:159) app//com.lmax.disruptor.BatchEventProcessor.run(BatchEventProcessor.java:125) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: master/692b295ad45c:0:becomeActiveMaster-MemStoreChunkPool Statistics java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: regionserver/692b295ad45c:0.procedureResultReporter java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.RemoteProcedureResultReporter.run(RemoteProcedureResultReporter.java:75) Potentially hanging thread: org.apache.hadoop.fs.FileSystem$Statistics$StatisticsDataReferenceCleaner java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:155) java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:176) app//org.apache.hadoop.fs.FileSystem$Statistics$StatisticsDataReferenceCleaner.run(FileSystem.java:4171) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ForkJoinPool.commonPool-worker-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.ForkJoinPool.awaitWork(ForkJoinPool.java:1724) java.base@17.0.11/java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1623) java.base@17.0.11/java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:165) Potentially hanging thread: SnapshotHandlerChoreCleaner java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-5 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-4 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HMaster-EventLoopGroup-1-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=337 (was 287) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=390 (was 424), ProcessCount=11 (was 11), AvailableMemoryMB=3781 (was 3271) - AvailableMemoryMB LEAK? -