2024-11-14 18:56:29,384 main DEBUG Apache Log4j Core 2.17.2 initializing configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@5745ca0e 2024-11-14 18:56:29,437 main DEBUG Took 0.050211 seconds to load 1 plugins from package org.apache.hadoop.hbase.logging 2024-11-14 18:56:29,437 main DEBUG PluginManager 'Core' found 129 plugins 2024-11-14 18:56:29,438 main DEBUG PluginManager 'Level' found 0 plugins 2024-11-14 18:56:29,440 main DEBUG PluginManager 'Lookup' found 16 plugins 2024-11-14 18:56:29,442 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-14 18:56:29,451 main DEBUG PluginManager 'TypeConverter' found 26 plugins 2024-11-14 18:56:29,470 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.util.MBeans", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-14 18:56:29,472 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-14 18:56:29,473 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.logging.TestJul2Slf4j", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-14 18:56:29,473 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-14 18:56:29,474 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.zookeeper", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-14 18:56:29,474 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-14 18:56:29,476 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSinkAdapter", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-14 18:56:29,476 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-14 18:56:29,477 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSystemImpl", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-14 18:56:29,478 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-14 18:56:29,481 main DEBUG LoggerConfig$Builder(additivity="false", level="WARN", levelAndRefs="null", name="org.apache.directory", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-14 18:56:29,482 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-14 18:56:29,483 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.ipc.FailedServers", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-14 18:56:29,483 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-14 18:56:29,484 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsConfig", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-14 18:56:29,484 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-14 18:56:29,485 main DEBUG LoggerConfig$Builder(additivity="null", level="INFO", levelAndRefs="null", name="org.apache.hadoop.hbase.ScheduledChore", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-14 18:56:29,486 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-14 18:56:29,486 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.regionserver.RSRpcServices", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-14 18:56:29,487 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-14 18:56:29,488 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-14 18:56:29,488 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-14 18:56:29,489 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-14 18:56:29,489 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-14 18:56:29,490 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hbase.thirdparty.io.netty.channel", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-14 18:56:29,490 main DEBUG Building Plugin[name=root, class=org.apache.logging.log4j.core.config.LoggerConfig$RootLogger]. 2024-11-14 18:56:29,492 main DEBUG LoggerConfig$RootLogger$Builder(additivity="null", level="null", levelAndRefs="INFO,Console", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-14 18:56:29,494 main DEBUG Building Plugin[name=loggers, class=org.apache.logging.log4j.core.config.LoggersPlugin]. 2024-11-14 18:56:29,496 main DEBUG createLoggers(={org.apache.hadoop.metrics2.util.MBeans, org.apache.hadoop.hbase.logging.TestJul2Slf4j, org.apache.zookeeper, org.apache.hadoop.metrics2.impl.MetricsSinkAdapter, org.apache.hadoop.metrics2.impl.MetricsSystemImpl, org.apache.directory, org.apache.hadoop.hbase.ipc.FailedServers, org.apache.hadoop.metrics2.impl.MetricsConfig, org.apache.hadoop.hbase.ScheduledChore, org.apache.hadoop.hbase.regionserver.RSRpcServices, org.apache.hadoop, org.apache.hadoop.hbase, org.apache.hbase.thirdparty.io.netty.channel, root}) 2024-11-14 18:56:29,497 main DEBUG Building Plugin[name=layout, class=org.apache.logging.log4j.core.layout.PatternLayout]. 2024-11-14 18:56:29,499 main DEBUG PatternLayout$Builder(pattern="%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n", PatternSelector=null, Configuration(PropertiesConfig), Replace=null, charset="null", alwaysWriteExceptions="null", disableAnsi="null", noConsoleNoAnsi="null", header="null", footer="null") 2024-11-14 18:56:29,500 main DEBUG PluginManager 'Converter' found 47 plugins 2024-11-14 18:56:29,512 main DEBUG Building Plugin[name=appender, class=org.apache.hadoop.hbase.logging.HBaseTestAppender]. 2024-11-14 18:56:29,515 main DEBUG HBaseTestAppender$Builder(target="SYSTEM_ERR", maxSize="1G", bufferedIo="null", bufferSize="null", immediateFlush="null", ignoreExceptions="null", PatternLayout(%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n), name="Console", Configuration(PropertiesConfig), Filter=null, ={}) 2024-11-14 18:56:29,517 main DEBUG Starting HBaseTestOutputStreamManager SYSTEM_ERR 2024-11-14 18:56:29,518 main DEBUG Building Plugin[name=appenders, class=org.apache.logging.log4j.core.config.AppendersPlugin]. 2024-11-14 18:56:29,518 main DEBUG createAppenders(={Console}) 2024-11-14 18:56:29,519 main DEBUG Configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@5745ca0e initialized 2024-11-14 18:56:29,520 main DEBUG Starting configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@5745ca0e 2024-11-14 18:56:29,520 main DEBUG Started configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@5745ca0e OK. 2024-11-14 18:56:29,521 main DEBUG Shutting down OutputStreamManager SYSTEM_OUT.false.false-1 2024-11-14 18:56:29,521 main DEBUG OutputStream closed 2024-11-14 18:56:29,521 main DEBUG Shut down OutputStreamManager SYSTEM_OUT.false.false-1, all resources released: true 2024-11-14 18:56:29,522 main DEBUG Appender DefaultConsole-1 stopped with status true 2024-11-14 18:56:29,522 main DEBUG Stopped org.apache.logging.log4j.core.config.DefaultConfiguration@74751b3 OK 2024-11-14 18:56:29,611 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6 2024-11-14 18:56:29,613 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=StatusLogger 2024-11-14 18:56:29,615 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=ContextSelector 2024-11-14 18:56:29,616 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name= 2024-11-14 18:56:29,617 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.directory 2024-11-14 18:56:29,618 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSinkAdapter 2024-11-14 18:56:29,618 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.zookeeper 2024-11-14 18:56:29,618 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.logging.TestJul2Slf4j 2024-11-14 18:56:29,619 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSystemImpl 2024-11-14 18:56:29,619 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.util.MBeans 2024-11-14 18:56:29,620 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase 2024-11-14 18:56:29,620 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop 2024-11-14 18:56:29,620 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ipc.FailedServers 2024-11-14 18:56:29,621 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.regionserver.RSRpcServices 2024-11-14 18:56:29,621 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsConfig 2024-11-14 18:56:29,621 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hbase.thirdparty.io.netty.channel 2024-11-14 18:56:29,622 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ScheduledChore 2024-11-14 18:56:29,623 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Appenders,name=Console 2024-11-14 18:56:29,625 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-14 18:56:29,625 main DEBUG Reconfiguration complete for context[name=1dbd16a6] at URI jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-logging/target/hbase-logging-3.0.0-beta-2-SNAPSHOT-tests.jar!/log4j2.properties (org.apache.logging.log4j.core.LoggerContext@528c868) with optional ClassLoader: null 2024-11-14 18:56:29,626 main DEBUG Shutdown hook enabled. Registering a new one. 2024-11-14 18:56:29,626 main DEBUG LoggerContext[name=1dbd16a6, org.apache.logging.log4j.core.LoggerContext@528c868] started OK. 2024-11-14T18:56:29,640 INFO [main {}] hbase.HBaseClassTestRule(94): Test class org.apache.hadoop.hbase.io.hfile.TestPrefetchRSClose timeout: 13 mins 2024-11-14 18:56:29,643 main DEBUG AsyncLogger.ThreadNameStrategy=UNCACHED (user specified null, default is UNCACHED) 2024-11-14 18:56:29,643 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-14T18:56:29,963 DEBUG [main {}] hbase.HBaseTestingUtil(323): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e5e0d570-5e1f-a378-f19c-4a0a374d5e72 2024-11-14T18:56:29,994 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: io.hfile.TestPrefetchRSClose#testPrefetchPersistence Thread=12, OpenFileDescriptor=287, MaxFileDescriptor=1048576, SystemLoadAverage=485, ProcessCount=11, AvailableMemoryMB=3957 2024-11-14T18:56:30,112 WARN [Time-limited test {}] util.NativeCodeLoader(60): Unable to load native-hadoop library for your platform... using builtin-java classes where applicable 2024-11-14T18:56:30,235 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-14T18:56:30,257 INFO [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e5e0d570-5e1f-a378-f19c-4a0a374d5e72/cluster_813d9243-ffee-ba6f-ef30-b3bffae974ef, deleteOnExit=true 2024-11-14T18:56:30,409 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e5e0d570-5e1f-a378-f19c-4a0a374d5e72/cluster_813d9243-ffee-ba6f-ef30-b3bffae974ef/zookeeper_0, clientPort=56960, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e5e0d570-5e1f-a378-f19c-4a0a374d5e72/cluster_813d9243-ffee-ba6f-ef30-b3bffae974ef/zookeeper_0/version-2, dataDirSize=457 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e5e0d570-5e1f-a378-f19c-4a0a374d5e72/cluster_813d9243-ffee-ba6f-ef30-b3bffae974ef/zookeeper_0/version-2, dataLogSize=457 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0 2024-11-14T18:56:30,421 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=56960 2024-11-14T18:56:30,422 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-14T18:56:30,513 INFO [Time-limited test {}] util.FSUtils(489): Created version file at /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e5e0d570-5e1f-a378-f19c-4a0a374d5e72/b8642a29-d0d1-92cd-8fbf-cfee23d7ae40 with version=8 2024-11-14T18:56:30,514 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1139): Setting hbase.fs.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e5e0d570-5e1f-a378-f19c-4a0a374d5e72/b8642a29-d0d1-92cd-8fbf-cfee23d7ae40/hbase-staging 2024-11-14T18:56:30,615 DEBUG [Time-limited test {}] channel.MultithreadEventLoopGroup(44): -Dio.netty.eventLoopThreads: 16 2024-11-14T18:56:30,928 INFO [Time-limited test {}] client.ConnectionUtils(128): master/853c158ffd74:0 server-side Connection retries=45 2024-11-14T18:56:30,938 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-14T18:56:30,940 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-14T18:56:30,948 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-14T18:56:30,949 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-14T18:56:30,949 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-14T18:56:31,128 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService 2024-11-14T18:56:31,229 WARN [Time-limited test {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-14T18:56:31,311 INFO [Time-limited test {}] metrics.MetricRegistriesLoader(60): Loaded MetricRegistries class org.apache.hadoop.hbase.metrics.impl.MetricRegistriesImpl 2024-11-14T18:56:31,326 DEBUG [Time-limited test {}] util.ClassSize(228): Using Unsafe to estimate memory layout 2024-11-14T18:56:31,331 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-14T18:56:31,376 DEBUG [Time-limited test {}] channel.DefaultChannelId(84): -Dio.netty.processId: 132392 (auto-detected) 2024-11-14T18:56:31,377 DEBUG [Time-limited test {}] channel.DefaultChannelId(106): -Dio.netty.machineId: 02:42:ac:ff:fe:11:00:02 (auto-detected) 2024-11-14T18:56:31,402 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:36053 2024-11-14T18:56:31,432 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:36053 connecting to ZooKeeper ensemble=127.0.0.1:56960 2024-11-14T18:56:31,630 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:360530x0, quorum=127.0.0.1:56960, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-14T18:56:31,635 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:36053-0x1013bfe4f770000 connected 2024-11-14T18:56:31,725 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-14T18:56:31,726 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-14T18:56:31,738 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:36053-0x1013bfe4f770000, quorum=127.0.0.1:56960, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-14T18:56:31,743 INFO [Time-limited test {}] master.HMaster(525): hbase.rootdir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e5e0d570-5e1f-a378-f19c-4a0a374d5e72/b8642a29-d0d1-92cd-8fbf-cfee23d7ae40, hbase.cluster.distributed=false 2024-11-14T18:56:31,777 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:36053-0x1013bfe4f770000, quorum=127.0.0.1:56960, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-14T18:56:31,787 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=36053 2024-11-14T18:56:31,787 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=36053 2024-11-14T18:56:31,788 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=36053 2024-11-14T18:56:31,789 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=36053 2024-11-14T18:56:31,789 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=36053 2024-11-14T18:56:31,900 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/853c158ffd74:0 server-side Connection retries=45 2024-11-14T18:56:31,902 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-14T18:56:31,903 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-14T18:56:31,903 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-14T18:56:31,903 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-14T18:56:31,904 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-14T18:56:31,907 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-14T18:56:31,911 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-14T18:56:31,915 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:41231 2024-11-14T18:56:31,918 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:41231 connecting to ZooKeeper ensemble=127.0.0.1:56960 2024-11-14T18:56:31,918 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-14T18:56:31,920 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-14T18:56:31,945 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:412310x0, quorum=127.0.0.1:56960, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-14T18:56:31,946 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:412310x0, quorum=127.0.0.1:56960, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-14T18:56:31,946 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:41231-0x1013bfe4f770001 connected 2024-11-14T18:56:31,953 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-14T18:56:31,969 INFO [Time-limited test {}] bucket.FileIOEngine(99): Allocating cache 400 MB, on the path:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e5e0d570-5e1f-a378-f19c-4a0a374d5e72/bucket.cache 2024-11-14T18:56:31,978 INFO [Time-limited test {}] bucket.BucketCache(365): Instantiating BucketCache with acceptableFactor: 0.95, minFactor: 0.85, extraFreeFactor: 0.1, singleFactor: 0.25, multiFactor: 0.5, memoryFactor: 0.25, useStrongRef: false 2024-11-14T18:56:31,980 INFO [Time-limited test {}] bucket.BucketCache(479): Starting BucketCachePersisterThread 2024-11-14T18:56:31,982 INFO [Time-limited test {}] bucket.BucketCachePersister(38): BucketCachePersister started with interval: 1000 2024-11-14T18:56:31,995 INFO [Thread-10 {}] bucket.BucketCache(1531): Started retrieving bucket cache from file 2024-11-14T18:56:31,995 WARN [Thread-10 {}] bucket.BucketCache(1534): Persistence file missing! It's ok if it's first run after enabling persistent cache. 2024-11-14T18:56:32,004 INFO [Time-limited test {}] bucket.BucketCache(404): Started bucket cache; ioengine=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e5e0d570-5e1f-a378-f19c-4a0a374d5e72/bucket.cache, capacity=400 MB, blockSize=64 KB, writerThreadNum=3, writerQLen=64, persistencePath=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e5e0d570-5e1f-a378-f19c-4a0a374d5e72/bucket.persistence, bucketAllocator=org.apache.hadoop.hbase.io.hfile.bucket.BucketAllocator 2024-11-14T18:56:32,006 INFO [Thread-10 {}] bucket.BucketAllocator(343): Cache totalSize=418148352, buckets=199, bucket capacity=2101248=(4*525312)=(FEWEST_ITEMS_IN_BUCKET*(largest configured bucketcache size)) 2024-11-14T18:56:32,007 INFO [Thread-10 {}] bucket.BucketCache(414): Persistent bucket cache recovery from /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e5e0d570-5e1f-a378-f19c-4a0a374d5e72/bucket.persistence is complete. 2024-11-14T18:56:32,018 DEBUG [Time-limited test {}] mob.MobFileCache(123): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-14T18:56:32,022 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:41231-0x1013bfe4f770001, quorum=127.0.0.1:56960, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-14T18:56:32,030 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:41231-0x1013bfe4f770001, quorum=127.0.0.1:56960, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-14T18:56:32,031 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=41231 2024-11-14T18:56:32,034 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=41231 2024-11-14T18:56:32,041 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=41231 2024-11-14T18:56:32,042 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=41231 2024-11-14T18:56:32,047 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=41231 2024-11-14T18:56:32,072 DEBUG [M:0;853c158ffd74:36053 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;853c158ffd74:36053 2024-11-14T18:56:32,079 INFO [master/853c158ffd74:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/853c158ffd74,36053,1731610590719 2024-11-14T18:56:32,116 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41231-0x1013bfe4f770001, quorum=127.0.0.1:56960, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-14T18:56:32,116 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36053-0x1013bfe4f770000, quorum=127.0.0.1:56960, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-14T18:56:32,119 DEBUG [master/853c158ffd74:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:36053-0x1013bfe4f770000, quorum=127.0.0.1:56960, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/853c158ffd74,36053,1731610590719 2024-11-14T18:56:32,178 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41231-0x1013bfe4f770001, quorum=127.0.0.1:56960, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-14T18:56:32,179 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36053-0x1013bfe4f770000, quorum=127.0.0.1:56960, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-14T18:56:32,179 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41231-0x1013bfe4f770001, quorum=127.0.0.1:56960, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-14T18:56:32,182 DEBUG [master/853c158ffd74:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:36053-0x1013bfe4f770000, quorum=127.0.0.1:56960, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-11-14T18:56:32,184 INFO [master/853c158ffd74:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/853c158ffd74,36053,1731610590719 from backup master directory 2024-11-14T18:56:32,195 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41231-0x1013bfe4f770001, quorum=127.0.0.1:56960, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-14T18:56:32,195 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36053-0x1013bfe4f770000, quorum=127.0.0.1:56960, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/853c158ffd74,36053,1731610590719 2024-11-14T18:56:32,195 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36053-0x1013bfe4f770000, quorum=127.0.0.1:56960, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-14T18:56:32,196 WARN [master/853c158ffd74:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-14T18:56:32,197 INFO [master/853c158ffd74:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=853c158ffd74,36053,1731610590719 2024-11-14T18:56:32,199 INFO [master/853c158ffd74:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating data MemStoreChunkPool with chunk size 2 MB, max count 396, initial count 0 2024-11-14T18:56:32,203 INFO [master/853c158ffd74:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating index MemStoreChunkPool with chunk size 204.80 KB, max count 440, initial count 0 2024-11-14T18:56:32,229 DEBUG [master/853c158ffd74:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e5e0d570-5e1f-a378-f19c-4a0a374d5e72/b8642a29-d0d1-92cd-8fbf-cfee23d7ae40/hbase.id] with ID: 9dea2c90-c8ab-45d3-9339-db2a9f803375 2024-11-14T18:56:32,229 DEBUG [master/853c158ffd74:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e5e0d570-5e1f-a378-f19c-4a0a374d5e72/b8642a29-d0d1-92cd-8fbf-cfee23d7ae40/.tmp/hbase.id 2024-11-14T18:56:32,241 DEBUG [master/853c158ffd74:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e5e0d570-5e1f-a378-f19c-4a0a374d5e72/b8642a29-d0d1-92cd-8fbf-cfee23d7ae40/.tmp/hbase.id]:[file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e5e0d570-5e1f-a378-f19c-4a0a374d5e72/b8642a29-d0d1-92cd-8fbf-cfee23d7ae40/hbase.id] 2024-11-14T18:56:32,335 DEBUG [master/853c158ffd74:0:becomeActiveMaster {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-14T18:56:32,342 INFO [master/853c158ffd74:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem. 2024-11-14T18:56:32,367 INFO [master/853c158ffd74:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 23ms. 2024-11-14T18:56:32,415 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36053-0x1013bfe4f770000, quorum=127.0.0.1:56960, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-14T18:56:32,416 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41231-0x1013bfe4f770001, quorum=127.0.0.1:56960, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-14T18:56:32,448 INFO [master/853c158ffd74:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-14T18:56:32,452 INFO [master/853c158ffd74:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-11-14T18:56:32,508 DEBUG [master/853c158ffd74:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(244): No decryptEncryptedDataEncryptionKey method in DFSClient, should be hadoop version with HDFS-12396 java.lang.NoSuchMethodException: org.apache.hadoop.hdfs.DFSClient.decryptEncryptedDataEncryptionKey(org.apache.hadoop.fs.FileEncryptionInfo) at java.lang.Class.getDeclaredMethod(Class.java:2675) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelperWithoutHDFS12396(FanOutOneBlockAsyncDFSOutputSaslHelper.java:183) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelper(FanOutOneBlockAsyncDFSOutputSaslHelper.java:242) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.(FanOutOneBlockAsyncDFSOutputSaslHelper.java:253) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.lang.Class.forName0(Native Method) ~[?:?] at java.lang.Class.forName(Class.java:375) ~[?:?] at org.apache.hadoop.hbase.wal.AsyncFSWALProvider.load(AsyncFSWALProvider.java:150) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.getProviderClass(WALFactory.java:174) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:262) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:231) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegion.create(MasterRegion.java:400) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegionFactory.create(MasterRegionFactory.java:135) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.finishActiveMasterInitialization(HMaster.java:1003) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.startActiveMasterManager(HMaster.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.lambda$run$0(HMaster.java:613) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.lambda$tracedRunnable$2(TraceUtil.java:155) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T18:56:32,517 INFO [master/853c158ffd74:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-14T18:56:32,633 INFO [master/853c158ffd74:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e5e0d570-5e1f-a378-f19c-4a0a374d5e72/b8642a29-d0d1-92cd-8fbf-cfee23d7ae40/MasterData/data/master/store 2024-11-14T18:56:32,689 INFO [master/853c158ffd74:0:becomeActiveMaster {}] throttle.StoreHotnessProtector(112): StoreHotnessProtector is disabled. Set hbase.region.store.parallel.put.limit > 0 to enable, which may help mitigate load under heavy write pressure. 2024-11-14T18:56:32,694 DEBUG [master/853c158ffd74:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-14T18:56:32,696 DEBUG [master/853c158ffd74:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-14T18:56:32,696 INFO [master/853c158ffd74:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-14T18:56:32,696 DEBUG [master/853c158ffd74:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-14T18:56:32,699 DEBUG [master/853c158ffd74:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-14T18:56:32,699 DEBUG [master/853c158ffd74:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-14T18:56:32,699 INFO [master/853c158ffd74:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-14T18:56:32,700 DEBUG [master/853c158ffd74:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1731610592696Disabling compacts and flushes for region at 1731610592696Disabling writes for close at 1731610592699 (+3 ms)Writing region close event to WAL at 1731610592699Closed at 1731610592699 2024-11-14T18:56:32,701 WARN [master/853c158ffd74:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e5e0d570-5e1f-a378-f19c-4a0a374d5e72/b8642a29-d0d1-92cd-8fbf-cfee23d7ae40/MasterData/data/master/store/.initializing 2024-11-14T18:56:32,702 DEBUG [master/853c158ffd74:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e5e0d570-5e1f-a378-f19c-4a0a374d5e72/b8642a29-d0d1-92cd-8fbf-cfee23d7ae40/MasterData/WALs/853c158ffd74,36053,1731610590719 2024-11-14T18:56:32,745 INFO [master/853c158ffd74:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-14T18:56:32,765 INFO [master/853c158ffd74:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=853c158ffd74%2C36053%2C1731610590719, suffix=, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e5e0d570-5e1f-a378-f19c-4a0a374d5e72/b8642a29-d0d1-92cd-8fbf-cfee23d7ae40/MasterData/WALs/853c158ffd74,36053,1731610590719, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e5e0d570-5e1f-a378-f19c-4a0a374d5e72/b8642a29-d0d1-92cd-8fbf-cfee23d7ae40/MasterData/oldWALs, maxLogs=10 2024-11-14T18:56:32,853 INFO [master/853c158ffd74:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e5e0d570-5e1f-a378-f19c-4a0a374d5e72/b8642a29-d0d1-92cd-8fbf-cfee23d7ae40/MasterData/WALs/853c158ffd74,36053,1731610590719/853c158ffd74%2C36053%2C1731610590719.1731610592772 2024-11-14T18:56:32,855 DEBUG [master/853c158ffd74:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-11-14T18:56:32,856 DEBUG [master/853c158ffd74:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-11-14T18:56:32,857 DEBUG [master/853c158ffd74:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-14T18:56:32,862 DEBUG [master/853c158ffd74:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-11-14T18:56:32,863 DEBUG [master/853c158ffd74:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-11-14T18:56:32,917 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=true, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-11-14T18:56:32,975 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-11-14T18:56:32,982 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T18:56:32,985 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-14T18:56:32,986 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=true, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-11-14T18:56:33,004 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-11-14T18:56:33,004 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T18:56:33,005 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T18:56:33,005 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=true, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-11-14T18:56:33,028 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-11-14T18:56:33,029 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T18:56:33,029 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T18:56:33,030 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=true, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-11-14T18:56:33,068 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-11-14T18:56:33,068 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T18:56:33,069 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T18:56:33,071 DEBUG [master/853c158ffd74:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682 2024-11-14T18:56:33,076 DEBUG [master/853c158ffd74:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e5e0d570-5e1f-a378-f19c-4a0a374d5e72/b8642a29-d0d1-92cd-8fbf-cfee23d7ae40/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-14T18:56:33,077 DEBUG [master/853c158ffd74:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e5e0d570-5e1f-a378-f19c-4a0a374d5e72/b8642a29-d0d1-92cd-8fbf-cfee23d7ae40/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-14T18:56:33,081 DEBUG [master/853c158ffd74:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682 2024-11-14T18:56:33,082 DEBUG [master/853c158ffd74:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682 2024-11-14T18:56:33,086 DEBUG [master/853c158ffd74:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-14T18:56:33,090 DEBUG [master/853c158ffd74:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-11-14T18:56:33,100 DEBUG [master/853c158ffd74:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e5e0d570-5e1f-a378-f19c-4a0a374d5e72/b8642a29-d0d1-92cd-8fbf-cfee23d7ae40/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-14T18:56:33,101 INFO [master/853c158ffd74:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=64958953, jitterRate=-0.03203617036342621}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-14T18:56:33,112 DEBUG [master/853c158ffd74:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682: Writing region info on filesystem at 1731610592882Initializing all the Stores at 1731610592882Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731610592883 (+1 ms)Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731610592885 (+2 ms)Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731610592887 (+2 ms)Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731610592887Cleaning up temporary data from old regions at 1731610593082 (+195 ms)Region opened successfully at 1731610593112 (+30 ms) 2024-11-14T18:56:33,114 INFO [master/853c158ffd74:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-11-14T18:56:33,162 DEBUG [master/853c158ffd74:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@8a30fe1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=853c158ffd74/172.17.0.2:0 2024-11-14T18:56:33,204 INFO [master/853c158ffd74:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating... 2024-11-14T18:56:33,223 INFO [master/853c158ffd74:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5 2024-11-14T18:56:33,223 INFO [master/853c158ffd74:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50 2024-11-14T18:56:33,228 INFO [master/853c158ffd74:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-14T18:56:33,229 INFO [master/853c158ffd74:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 1 msec 2024-11-14T18:56:33,240 INFO [master/853c158ffd74:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 10 msec 2024-11-14T18:56:33,241 INFO [master/853c158ffd74:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-11-14T18:56:33,287 INFO [master/853c158ffd74:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2024-11-14T18:56:33,301 DEBUG [master/853c158ffd74:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:36053-0x1013bfe4f770000, quorum=127.0.0.1:56960, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error) 2024-11-14T18:56:33,365 DEBUG [master/853c158ffd74:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false 2024-11-14T18:56:33,368 INFO [master/853c158ffd74:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2024-11-14T18:56:33,370 DEBUG [master/853c158ffd74:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:36053-0x1013bfe4f770000, quorum=127.0.0.1:56960, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error) 2024-11-14T18:56:33,382 DEBUG [master/853c158ffd74:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false 2024-11-14T18:56:33,385 INFO [master/853c158ffd74:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2024-11-14T18:56:33,390 DEBUG [master/853c158ffd74:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:36053-0x1013bfe4f770000, quorum=127.0.0.1:56960, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error) 2024-11-14T18:56:33,403 DEBUG [master/853c158ffd74:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false 2024-11-14T18:56:33,405 DEBUG [master/853c158ffd74:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:36053-0x1013bfe4f770000, quorum=127.0.0.1:56960, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error) 2024-11-14T18:56:33,415 DEBUG [master/853c158ffd74:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false 2024-11-14T18:56:33,443 DEBUG [master/853c158ffd74:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:36053-0x1013bfe4f770000, quorum=127.0.0.1:56960, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error) 2024-11-14T18:56:33,453 DEBUG [master/853c158ffd74:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false 2024-11-14T18:56:33,478 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36053-0x1013bfe4f770000, quorum=127.0.0.1:56960, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-14T18:56:33,478 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36053-0x1013bfe4f770000, quorum=127.0.0.1:56960, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-14T18:56:33,479 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41231-0x1013bfe4f770001, quorum=127.0.0.1:56960, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-14T18:56:33,479 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41231-0x1013bfe4f770001, quorum=127.0.0.1:56960, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-14T18:56:33,483 INFO [master/853c158ffd74:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=853c158ffd74,36053,1731610590719, sessionid=0x1013bfe4f770000, setting cluster-up flag (Was=false) 2024-11-14T18:56:33,589 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41231-0x1013bfe4f770001, quorum=127.0.0.1:56960, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-14T18:56:33,589 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36053-0x1013bfe4f770000, quorum=127.0.0.1:56960, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-14T18:56:33,623 DEBUG [master/853c158ffd74:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort 2024-11-14T18:56:33,626 DEBUG [master/853c158ffd74:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=853c158ffd74,36053,1731610590719 2024-11-14T18:56:33,644 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41231-0x1013bfe4f770001, quorum=127.0.0.1:56960, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-14T18:56:33,645 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36053-0x1013bfe4f770000, quorum=127.0.0.1:56960, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-14T18:56:33,673 DEBUG [master/853c158ffd74:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort 2024-11-14T18:56:33,675 DEBUG [master/853c158ffd74:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=853c158ffd74,36053,1731610590719 2024-11-14T18:56:33,681 INFO [master/853c158ffd74:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e5e0d570-5e1f-a378-f19c-4a0a374d5e72/b8642a29-d0d1-92cd-8fbf-cfee23d7ae40/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again 2024-11-14T18:56:33,755 INFO [RS:0;853c158ffd74:41231 {}] regionserver.HRegionServer(746): ClusterId : 9dea2c90-c8ab-45d3-9339-db2a9f803375 2024-11-14T18:56:33,757 DEBUG [RS:0;853c158ffd74:41231 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-14T18:56:33,765 DEBUG [master/853c158ffd74:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-14T18:56:33,779 INFO [master/853c158ffd74:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2 2024-11-14T18:56:33,801 INFO [master/853c158ffd74:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2024-11-14T18:56:33,801 DEBUG [RS:0;853c158ffd74:41231 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-14T18:56:33,801 DEBUG [RS:0;853c158ffd74:41231 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-14T18:56:33,809 DEBUG [master/853c158ffd74:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: 853c158ffd74,36053,1731610590719 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2024-11-14T18:56:33,814 DEBUG [RS:0;853c158ffd74:41231 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-14T18:56:33,815 DEBUG [RS:0;853c158ffd74:41231 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@17ce342, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=853c158ffd74/172.17.0.2:0 2024-11-14T18:56:33,820 DEBUG [master/853c158ffd74:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/853c158ffd74:0, corePoolSize=5, maxPoolSize=5 2024-11-14T18:56:33,820 DEBUG [master/853c158ffd74:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/853c158ffd74:0, corePoolSize=5, maxPoolSize=5 2024-11-14T18:56:33,821 DEBUG [master/853c158ffd74:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/853c158ffd74:0, corePoolSize=5, maxPoolSize=5 2024-11-14T18:56:33,821 DEBUG [master/853c158ffd74:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/853c158ffd74:0, corePoolSize=5, maxPoolSize=5 2024-11-14T18:56:33,821 DEBUG [master/853c158ffd74:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/853c158ffd74:0, corePoolSize=10, maxPoolSize=10 2024-11-14T18:56:33,821 DEBUG [master/853c158ffd74:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/853c158ffd74:0, corePoolSize=1, maxPoolSize=1 2024-11-14T18:56:33,821 DEBUG [master/853c158ffd74:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/853c158ffd74:0, corePoolSize=2, maxPoolSize=2 2024-11-14T18:56:33,822 DEBUG [master/853c158ffd74:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/853c158ffd74:0, corePoolSize=1, maxPoolSize=1 2024-11-14T18:56:33,827 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-14T18:56:33,828 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region 2024-11-14T18:56:33,828 INFO [master/853c158ffd74:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731610623828 2024-11-14T18:56:33,830 INFO [master/853c158ffd74:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2024-11-14T18:56:33,831 INFO [master/853c158ffd74:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2024-11-14T18:56:33,833 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T18:56:33,833 INFO [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-11-14T18:56:33,835 INFO [master/853c158ffd74:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2024-11-14T18:56:33,835 INFO [master/853c158ffd74:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2024-11-14T18:56:33,836 INFO [master/853c158ffd74:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2024-11-14T18:56:33,836 DEBUG [RS:0;853c158ffd74:41231 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;853c158ffd74:41231 2024-11-14T18:56:33,836 INFO [master/853c158ffd74:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2024-11-14T18:56:33,837 INFO [master/853c158ffd74:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-14T18:56:33,841 INFO [master/853c158ffd74:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2024-11-14T18:56:33,841 INFO [RS:0;853c158ffd74:41231 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-14T18:56:33,841 INFO [RS:0;853c158ffd74:41231 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-14T18:56:33,841 DEBUG [RS:0;853c158ffd74:41231 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-14T18:56:33,842 INFO [master/853c158ffd74:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2024-11-14T18:56:33,843 INFO [master/853c158ffd74:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2024-11-14T18:56:33,845 INFO [RS:0;853c158ffd74:41231 {}] regionserver.HRegionServer(2659): reportForDuty to master=853c158ffd74,36053,1731610590719 with port=41231, startcode=1731610591853 2024-11-14T18:56:33,851 INFO [master/853c158ffd74:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2024-11-14T18:56:33,851 INFO [master/853c158ffd74:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2024-11-14T18:56:33,853 DEBUG [master/853c158ffd74:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/853c158ffd74:0:becomeActiveMaster-HFileCleaner.large.0-1731610593853,5,FailOnTimeoutGroup] 2024-11-14T18:56:33,854 DEBUG [master/853c158ffd74:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/853c158ffd74:0:becomeActiveMaster-HFileCleaner.small.0-1731610593854,5,FailOnTimeoutGroup] 2024-11-14T18:56:33,854 INFO [master/853c158ffd74:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-14T18:56:33,854 INFO [master/853c158ffd74:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2024-11-14T18:56:33,855 INFO [master/853c158ffd74:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2024-11-14T18:56:33,856 INFO [master/853c158ffd74:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2024-11-14T18:56:33,857 DEBUG [RS:0;853c158ffd74:41231 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-14T18:56:33,860 INFO [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e5e0d570-5e1f-a378-f19c-4a0a374d5e72/b8642a29-d0d1-92cd-8fbf-cfee23d7ae40/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321 2024-11-14T18:56:33,860 INFO [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e5e0d570-5e1f-a378-f19c-4a0a374d5e72/b8642a29-d0d1-92cd-8fbf-cfee23d7ae40 2024-11-14T18:56:33,869 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-14T18:56:33,871 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=true, for column family info of region 1588230740 2024-11-14T18:56:33,875 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-14T18:56:33,875 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T18:56:33,876 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-14T18:56:33,876 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=true, for column family ns of region 1588230740 2024-11-14T18:56:33,880 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-14T18:56:33,880 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T18:56:33,880 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-14T18:56:33,881 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=true, for column family rep_barrier of region 1588230740 2024-11-14T18:56:33,891 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-14T18:56:33,892 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T18:56:33,892 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-14T18:56:33,893 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=true, for column family table of region 1588230740 2024-11-14T18:56:33,903 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-14T18:56:33,903 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T18:56:33,904 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-14T18:56:33,904 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-14T18:56:33,905 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e5e0d570-5e1f-a378-f19c-4a0a374d5e72/b8642a29-d0d1-92cd-8fbf-cfee23d7ae40/data/hbase/meta/1588230740 2024-11-14T18:56:33,905 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e5e0d570-5e1f-a378-f19c-4a0a374d5e72/b8642a29-d0d1-92cd-8fbf-cfee23d7ae40/data/hbase/meta/1588230740 2024-11-14T18:56:33,906 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-14T18:56:33,906 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-14T18:56:33,906 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-14T18:56:33,912 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-14T18:56:33,934 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e5e0d570-5e1f-a378-f19c-4a0a374d5e72/b8642a29-d0d1-92cd-8fbf-cfee23d7ae40/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-14T18:56:33,935 INFO [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=62006032, jitterRate=-0.07603812217712402}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-14T18:56:33,936 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36211, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=RegionServerStatusService 2024-11-14T18:56:33,940 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740: Writing region info on filesystem at 1731610593870Initializing all the Stores at 1731610593870Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731610593870Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731610593870Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731610593870Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731610593870Cleaning up temporary data from old regions at 1731610593906 (+36 ms)Region opened successfully at 1731610593939 (+33 ms) 2024-11-14T18:56:33,941 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-14T18:56:33,941 INFO [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-14T18:56:33,941 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-14T18:56:33,942 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-14T18:56:33,942 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-14T18:56:33,943 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=36053 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 853c158ffd74,41231,1731610591853 2024-11-14T18:56:33,944 INFO [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-14T18:56:33,944 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1731610593941Disabling compacts and flushes for region at 1731610593941Disabling writes for close at 1731610593942 (+1 ms)Writing region close event to WAL at 1731610593944 (+2 ms)Closed at 1731610593944 2024-11-14T18:56:33,946 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=36053 {}] master.ServerManager(517): Registering regionserver=853c158ffd74,41231,1731610591853 2024-11-14T18:56:33,951 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-14T18:56:33,951 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta 2024-11-14T18:56:33,958 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-11-14T18:56:33,962 DEBUG [RS:0;853c158ffd74:41231 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e5e0d570-5e1f-a378-f19c-4a0a374d5e72/b8642a29-d0d1-92cd-8fbf-cfee23d7ae40 2024-11-14T18:56:33,962 DEBUG [RS:0;853c158ffd74:41231 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=file:/ 2024-11-14T18:56:33,962 DEBUG [RS:0;853c158ffd74:41231 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-14T18:56:33,967 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-14T18:56:33,969 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2024-11-14T18:56:33,990 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36053-0x1013bfe4f770000, quorum=127.0.0.1:56960, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-14T18:56:33,991 DEBUG [RS:0;853c158ffd74:41231 {}] zookeeper.ZKUtil(111): regionserver:41231-0x1013bfe4f770001, quorum=127.0.0.1:56960, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/853c158ffd74,41231,1731610591853 2024-11-14T18:56:33,991 WARN [RS:0;853c158ffd74:41231 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-14T18:56:33,991 INFO [RS:0;853c158ffd74:41231 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-14T18:56:33,991 DEBUG [RS:0;853c158ffd74:41231 {}] regionserver.HRegionServer(1793): logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e5e0d570-5e1f-a378-f19c-4a0a374d5e72/b8642a29-d0d1-92cd-8fbf-cfee23d7ae40/WALs/853c158ffd74,41231,1731610591853 2024-11-14T18:56:33,994 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [853c158ffd74,41231,1731610591853] 2024-11-14T18:56:34,019 INFO [RS:0;853c158ffd74:41231 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-14T18:56:34,039 INFO [RS:0;853c158ffd74:41231 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-14T18:56:34,044 INFO [RS:0;853c158ffd74:41231 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-14T18:56:34,045 INFO [RS:0;853c158ffd74:41231 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-14T18:56:34,046 INFO [RS:0;853c158ffd74:41231 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-14T18:56:34,051 INFO [RS:0;853c158ffd74:41231 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-14T18:56:34,052 INFO [RS:0;853c158ffd74:41231 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-14T18:56:34,053 DEBUG [RS:0;853c158ffd74:41231 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/853c158ffd74:0, corePoolSize=1, maxPoolSize=1 2024-11-14T18:56:34,053 DEBUG [RS:0;853c158ffd74:41231 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/853c158ffd74:0, corePoolSize=1, maxPoolSize=1 2024-11-14T18:56:34,053 DEBUG [RS:0;853c158ffd74:41231 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/853c158ffd74:0, corePoolSize=1, maxPoolSize=1 2024-11-14T18:56:34,053 DEBUG [RS:0;853c158ffd74:41231 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/853c158ffd74:0, corePoolSize=1, maxPoolSize=1 2024-11-14T18:56:34,053 DEBUG [RS:0;853c158ffd74:41231 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/853c158ffd74:0, corePoolSize=1, maxPoolSize=1 2024-11-14T18:56:34,053 DEBUG [RS:0;853c158ffd74:41231 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/853c158ffd74:0, corePoolSize=2, maxPoolSize=2 2024-11-14T18:56:34,053 DEBUG [RS:0;853c158ffd74:41231 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/853c158ffd74:0, corePoolSize=1, maxPoolSize=1 2024-11-14T18:56:34,054 DEBUG [RS:0;853c158ffd74:41231 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/853c158ffd74:0, corePoolSize=1, maxPoolSize=1 2024-11-14T18:56:34,054 DEBUG [RS:0;853c158ffd74:41231 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/853c158ffd74:0, corePoolSize=1, maxPoolSize=1 2024-11-14T18:56:34,054 DEBUG [RS:0;853c158ffd74:41231 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/853c158ffd74:0, corePoolSize=1, maxPoolSize=1 2024-11-14T18:56:34,054 DEBUG [RS:0;853c158ffd74:41231 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/853c158ffd74:0, corePoolSize=1, maxPoolSize=1 2024-11-14T18:56:34,054 DEBUG [RS:0;853c158ffd74:41231 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/853c158ffd74:0, corePoolSize=1, maxPoolSize=1 2024-11-14T18:56:34,054 DEBUG [RS:0;853c158ffd74:41231 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/853c158ffd74:0, corePoolSize=3, maxPoolSize=3 2024-11-14T18:56:34,054 DEBUG [RS:0;853c158ffd74:41231 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/853c158ffd74:0, corePoolSize=3, maxPoolSize=3 2024-11-14T18:56:34,056 INFO [RS:0;853c158ffd74:41231 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-14T18:56:34,056 INFO [RS:0;853c158ffd74:41231 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-14T18:56:34,056 INFO [RS:0;853c158ffd74:41231 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-14T18:56:34,057 INFO [RS:0;853c158ffd74:41231 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-14T18:56:34,057 INFO [RS:0;853c158ffd74:41231 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-14T18:56:34,057 INFO [RS:0;853c158ffd74:41231 {}] hbase.ChoreService(168): Chore ScheduledChore name=853c158ffd74,41231,1731610591853-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-14T18:56:34,081 INFO [RS:0;853c158ffd74:41231 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-14T18:56:34,083 INFO [RS:0;853c158ffd74:41231 {}] hbase.ChoreService(168): Chore ScheduledChore name=853c158ffd74,41231,1731610591853-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-14T18:56:34,084 INFO [RS:0;853c158ffd74:41231 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-14T18:56:34,084 INFO [RS:0;853c158ffd74:41231 {}] regionserver.Replication(171): 853c158ffd74,41231,1731610591853 started 2024-11-14T18:56:34,111 INFO [RS:0;853c158ffd74:41231 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-14T18:56:34,111 INFO [RS:0;853c158ffd74:41231 {}] regionserver.HRegionServer(1482): Serving as 853c158ffd74,41231,1731610591853, RpcServer on 853c158ffd74/172.17.0.2:41231, sessionid=0x1013bfe4f770001 2024-11-14T18:56:34,112 DEBUG [RS:0;853c158ffd74:41231 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-14T18:56:34,113 DEBUG [RS:0;853c158ffd74:41231 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 853c158ffd74,41231,1731610591853 2024-11-14T18:56:34,113 DEBUG [RS:0;853c158ffd74:41231 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '853c158ffd74,41231,1731610591853' 2024-11-14T18:56:34,113 DEBUG [RS:0;853c158ffd74:41231 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-14T18:56:34,115 DEBUG [RS:0;853c158ffd74:41231 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-14T18:56:34,116 DEBUG [RS:0;853c158ffd74:41231 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-14T18:56:34,116 DEBUG [RS:0;853c158ffd74:41231 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-14T18:56:34,116 DEBUG [RS:0;853c158ffd74:41231 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 853c158ffd74,41231,1731610591853 2024-11-14T18:56:34,116 DEBUG [RS:0;853c158ffd74:41231 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '853c158ffd74,41231,1731610591853' 2024-11-14T18:56:34,116 DEBUG [RS:0;853c158ffd74:41231 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-14T18:56:34,117 DEBUG [RS:0;853c158ffd74:41231 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-14T18:56:34,118 DEBUG [RS:0;853c158ffd74:41231 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-14T18:56:34,118 INFO [RS:0;853c158ffd74:41231 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-14T18:56:34,118 INFO [RS:0;853c158ffd74:41231 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-14T18:56:34,120 WARN [853c158ffd74:36053 {}] assignment.AssignmentManager(2443): No servers available; cannot place 1 unassigned regions. 2024-11-14T18:56:34,228 INFO [RS:0;853c158ffd74:41231 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-14T18:56:34,230 INFO [RS:0;853c158ffd74:41231 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=853c158ffd74%2C41231%2C1731610591853, suffix=, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e5e0d570-5e1f-a378-f19c-4a0a374d5e72/b8642a29-d0d1-92cd-8fbf-cfee23d7ae40/WALs/853c158ffd74,41231,1731610591853, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e5e0d570-5e1f-a378-f19c-4a0a374d5e72/b8642a29-d0d1-92cd-8fbf-cfee23d7ae40/oldWALs, maxLogs=55 2024-11-14T18:56:34,273 INFO [RS:0;853c158ffd74:41231 {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e5e0d570-5e1f-a378-f19c-4a0a374d5e72/b8642a29-d0d1-92cd-8fbf-cfee23d7ae40/WALs/853c158ffd74,41231,1731610591853/853c158ffd74%2C41231%2C1731610591853.1731610594235 2024-11-14T18:56:34,274 DEBUG [RS:0;853c158ffd74:41231 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-11-14T18:56:34,373 DEBUG [853c158ffd74:36053 {}] assignment.AssignmentManager(2464): Processing assignQueue; systemServersCount=1, allServersCount=1 2024-11-14T18:56:34,387 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=853c158ffd74,41231,1731610591853 2024-11-14T18:56:34,393 INFO [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 853c158ffd74,41231,1731610591853, state=OPENING 2024-11-14T18:56:34,448 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2024-11-14T18:56:34,456 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36053-0x1013bfe4f770000, quorum=127.0.0.1:56960, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-14T18:56:34,456 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41231-0x1013bfe4f770001, quorum=127.0.0.1:56960, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-14T18:56:34,458 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-14T18:56:34,458 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-14T18:56:34,459 DEBUG [PEWorker-3 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-14T18:56:34,461 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=853c158ffd74,41231,1731610591853}] 2024-11-14T18:56:34,633 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-14T18:56:34,637 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45379, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-14T18:56:34,646 INFO [RS_OPEN_META-regionserver/853c158ffd74:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-11-14T18:56:34,646 INFO [RS_OPEN_META-regionserver/853c158ffd74:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-14T18:56:34,647 INFO [RS_OPEN_META-regionserver/853c158ffd74:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-11-14T18:56:34,648 INFO [RS_OPEN_META-regionserver/853c158ffd74:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=853c158ffd74%2C41231%2C1731610591853.meta, suffix=.meta, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e5e0d570-5e1f-a378-f19c-4a0a374d5e72/b8642a29-d0d1-92cd-8fbf-cfee23d7ae40/WALs/853c158ffd74,41231,1731610591853, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e5e0d570-5e1f-a378-f19c-4a0a374d5e72/b8642a29-d0d1-92cd-8fbf-cfee23d7ae40/oldWALs, maxLogs=55 2024-11-14T18:56:34,668 INFO [RS_OPEN_META-regionserver/853c158ffd74:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e5e0d570-5e1f-a378-f19c-4a0a374d5e72/b8642a29-d0d1-92cd-8fbf-cfee23d7ae40/WALs/853c158ffd74,41231,1731610591853/853c158ffd74%2C41231%2C1731610591853.meta.1731610594650.meta 2024-11-14T18:56:34,668 DEBUG [RS_OPEN_META-regionserver/853c158ffd74:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-11-14T18:56:34,669 DEBUG [RS_OPEN_META-regionserver/853c158ffd74:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-11-14T18:56:34,671 DEBUG [RS_OPEN_META-regionserver/853c158ffd74:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-11-14T18:56:34,675 DEBUG [RS_OPEN_META-regionserver/853c158ffd74:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-11-14T18:56:34,682 INFO [RS_OPEN_META-regionserver/853c158ffd74:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-11-14T18:56:34,687 DEBUG [RS_OPEN_META-regionserver/853c158ffd74:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-11-14T18:56:34,688 DEBUG [RS_OPEN_META-regionserver/853c158ffd74:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-14T18:56:34,688 DEBUG [RS_OPEN_META-regionserver/853c158ffd74:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-11-14T18:56:34,688 DEBUG [RS_OPEN_META-regionserver/853c158ffd74:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-11-14T18:56:34,690 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=true, for column family info of region 1588230740 2024-11-14T18:56:34,692 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-14T18:56:34,692 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T18:56:34,693 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-14T18:56:34,693 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=true, for column family ns of region 1588230740 2024-11-14T18:56:34,694 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-14T18:56:34,694 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T18:56:34,695 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-14T18:56:34,695 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=true, for column family rep_barrier of region 1588230740 2024-11-14T18:56:34,697 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-14T18:56:34,697 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T18:56:34,697 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-14T18:56:34,697 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=true, for column family table of region 1588230740 2024-11-14T18:56:34,698 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-14T18:56:34,699 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T18:56:34,699 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-14T18:56:34,699 DEBUG [RS_OPEN_META-regionserver/853c158ffd74:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-14T18:56:34,700 DEBUG [RS_OPEN_META-regionserver/853c158ffd74:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e5e0d570-5e1f-a378-f19c-4a0a374d5e72/b8642a29-d0d1-92cd-8fbf-cfee23d7ae40/data/hbase/meta/1588230740 2024-11-14T18:56:34,701 DEBUG [RS_OPEN_META-regionserver/853c158ffd74:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e5e0d570-5e1f-a378-f19c-4a0a374d5e72/b8642a29-d0d1-92cd-8fbf-cfee23d7ae40/data/hbase/meta/1588230740 2024-11-14T18:56:34,701 DEBUG [RS_OPEN_META-regionserver/853c158ffd74:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-14T18:56:34,702 DEBUG [RS_OPEN_META-regionserver/853c158ffd74:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-14T18:56:34,702 DEBUG [RS_OPEN_META-regionserver/853c158ffd74:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-14T18:56:34,703 DEBUG [RS_OPEN_META-regionserver/853c158ffd74:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-14T18:56:34,704 INFO [RS_OPEN_META-regionserver/853c158ffd74:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=58982547, jitterRate=-0.1210915595293045}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-14T18:56:34,704 DEBUG [RS_OPEN_META-regionserver/853c158ffd74:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-11-14T18:56:34,705 DEBUG [RS_OPEN_META-regionserver/853c158ffd74:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1731610594689Writing region info on filesystem at 1731610594689Initializing all the Stores at 1731610594690 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731610594690Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731610594690Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731610594690Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731610594690Cleaning up temporary data from old regions at 1731610594702 (+12 ms)Running coprocessor post-open hooks at 1731610594704 (+2 ms)Region opened successfully at 1731610594705 (+1 ms) 2024-11-14T18:56:34,713 INFO [RS_OPEN_META-regionserver/853c158ffd74:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1731610594625 2024-11-14T18:56:34,727 DEBUG [RS_OPEN_META-regionserver/853c158ffd74:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-11-14T18:56:34,728 INFO [RS_OPEN_META-regionserver/853c158ffd74:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-11-14T18:56:34,729 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=853c158ffd74,41231,1731610591853 2024-11-14T18:56:34,731 INFO [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 853c158ffd74,41231,1731610591853, state=OPEN 2024-11-14T18:56:34,788 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41231-0x1013bfe4f770001, quorum=127.0.0.1:56960, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-14T18:56:34,788 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36053-0x1013bfe4f770000, quorum=127.0.0.1:56960, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-14T18:56:34,788 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-14T18:56:34,788 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-14T18:56:34,788 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=853c158ffd74,41231,1731610591853 2024-11-14T18:56:34,792 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2 2024-11-14T18:56:34,792 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=853c158ffd74,41231,1731610591853 in 328 msec 2024-11-14T18:56:34,796 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1 2024-11-14T18:56:34,796 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 835 msec 2024-11-14T18:56:34,797 DEBUG [PEWorker-2 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-14T18:56:34,797 INFO [PEWorker-2 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces 2024-11-14T18:56:34,821 DEBUG [PEWorker-2 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-14T18:56:34,822 DEBUG [PEWorker-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=853c158ffd74,41231,1731610591853, seqNum=-1] 2024-11-14T18:56:34,845 DEBUG [PEWorker-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-14T18:56:34,847 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35099, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-14T18:56:34,863 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 1.1490 sec 2024-11-14T18:56:34,864 INFO [master/853c158ffd74:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1731610594864, completionTime=-1 2024-11-14T18:56:34,866 INFO [master/853c158ffd74:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=1; waited=0ms, expected min=1 server(s), max=1 server(s), master is running 2024-11-14T18:56:34,867 DEBUG [master/853c158ffd74:0:becomeActiveMaster {}] assignment.AssignmentManager(1756): Joining cluster... 2024-11-14T18:56:34,890 INFO [master/853c158ffd74:0:becomeActiveMaster {}] assignment.AssignmentManager(1768): Number of RegionServers=1 2024-11-14T18:56:34,890 INFO [master/853c158ffd74:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731610654890 2024-11-14T18:56:34,890 INFO [master/853c158ffd74:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731610714890 2024-11-14T18:56:34,890 INFO [master/853c158ffd74:0:becomeActiveMaster {}] assignment.AssignmentManager(1775): Joined the cluster in 23 msec 2024-11-14T18:56:34,893 INFO [master/853c158ffd74:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=853c158ffd74,36053,1731610590719-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-14T18:56:34,893 INFO [master/853c158ffd74:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=853c158ffd74,36053,1731610590719-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-14T18:56:34,893 INFO [master/853c158ffd74:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=853c158ffd74,36053,1731610590719-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-14T18:56:34,894 INFO [master/853c158ffd74:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-853c158ffd74:36053, period=300000, unit=MILLISECONDS is enabled. 2024-11-14T18:56:34,895 INFO [master/853c158ffd74:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2024-11-14T18:56:34,895 INFO [master/853c158ffd74:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled. 2024-11-14T18:56:34,901 DEBUG [master/853c158ffd74:0.Chore.1 {}] janitor.CatalogJanitor(180): 2024-11-14T18:56:34,921 INFO [master/853c158ffd74:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 2.723sec 2024-11-14T18:56:34,922 INFO [master/853c158ffd74:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2024-11-14T18:56:34,923 INFO [master/853c158ffd74:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2024-11-14T18:56:34,924 INFO [master/853c158ffd74:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2024-11-14T18:56:34,924 INFO [master/853c158ffd74:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2024-11-14T18:56:34,924 INFO [master/853c158ffd74:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2024-11-14T18:56:34,925 INFO [master/853c158ffd74:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=853c158ffd74,36053,1731610590719-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-14T18:56:34,925 INFO [master/853c158ffd74:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=853c158ffd74,36053,1731610590719-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2024-11-14T18:56:34,929 DEBUG [master/853c158ffd74:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds 2024-11-14T18:56:34,930 INFO [master/853c158ffd74:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2024-11-14T18:56:34,930 INFO [master/853c158ffd74:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=853c158ffd74,36053,1731610590719-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-14T18:56:34,965 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6316a2b4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T18:56:34,967 DEBUG [Time-limited test {}] nio.NioEventLoop(110): -Dio.netty.noKeySetOptimization: false 2024-11-14T18:56:34,967 DEBUG [Time-limited test {}] nio.NioEventLoop(111): -Dio.netty.selectorAutoRebuildThreshold: 512 2024-11-14T18:56:34,971 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 853c158ffd74,36053,-1 for getting cluster id 2024-11-14T18:56:34,973 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-14T18:56:34,981 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '9dea2c90-c8ab-45d3-9339-db2a9f803375' 2024-11-14T18:56:34,984 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-14T18:56:34,985 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "9dea2c90-c8ab-45d3-9339-db2a9f803375" 2024-11-14T18:56:34,987 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@40dd5b4f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T18:56:34,987 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [853c158ffd74,36053,-1] 2024-11-14T18:56:35,002 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-14T18:56:35,005 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T18:56:35,006 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52466, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-14T18:56:35,009 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7d03aff4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T18:56:35,009 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-14T18:56:35,019 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=853c158ffd74,41231,1731610591853, seqNum=-1] 2024-11-14T18:56:35,019 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-14T18:56:35,021 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37508, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-14T18:56:35,044 INFO [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=853c158ffd74,36053,1731610590719 2024-11-14T18:56:35,049 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-14T18:56:35,057 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncConnectionImpl(321): The fetched master address is 853c158ffd74,36053,1731610590719 2024-11-14T18:56:35,060 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@7fb6c939 2024-11-14T18:56:35,061 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-14T18:56:35,064 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52470, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-14T18:56:35,070 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36053 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'table1', {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-14T18:56:35,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36053 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=table1 2024-11-14T18:56:35,081 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=table1 execute state=CREATE_TABLE_PRE_OPERATION 2024-11-14T18:56:35,083 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36053 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "table1" procId is: 4 2024-11-14T18:56:35,084 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T18:56:35,085 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=table1 execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-14T18:56:35,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36053 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-14T18:56:35,099 INFO [RegionOpenAndInit-table1-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => ef8dfce57d5182c0203767dc7e68adf6, NAME => 'table1,,1731610595066.ef8dfce57d5182c0203767dc7e68adf6.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='table1', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e5e0d570-5e1f-a378-f19c-4a0a374d5e72/b8642a29-d0d1-92cd-8fbf-cfee23d7ae40 2024-11-14T18:56:35,107 DEBUG [RegionOpenAndInit-table1-pool-0 {}] regionserver.HRegion(898): Instantiated table1,,1731610595066.ef8dfce57d5182c0203767dc7e68adf6.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-14T18:56:35,107 DEBUG [RegionOpenAndInit-table1-pool-0 {}] regionserver.HRegion(1722): Closing ef8dfce57d5182c0203767dc7e68adf6, disabling compactions & flushes 2024-11-14T18:56:35,108 INFO [RegionOpenAndInit-table1-pool-0 {}] regionserver.HRegion(1755): Closing region table1,,1731610595066.ef8dfce57d5182c0203767dc7e68adf6. 2024-11-14T18:56:35,108 DEBUG [RegionOpenAndInit-table1-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on table1,,1731610595066.ef8dfce57d5182c0203767dc7e68adf6. 2024-11-14T18:56:35,108 DEBUG [RegionOpenAndInit-table1-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on table1,,1731610595066.ef8dfce57d5182c0203767dc7e68adf6. after waiting 0 ms 2024-11-14T18:56:35,108 DEBUG [RegionOpenAndInit-table1-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region table1,,1731610595066.ef8dfce57d5182c0203767dc7e68adf6. 2024-11-14T18:56:35,108 INFO [RegionOpenAndInit-table1-pool-0 {}] regionserver.HRegion(1973): Closed table1,,1731610595066.ef8dfce57d5182c0203767dc7e68adf6. 2024-11-14T18:56:35,108 DEBUG [RegionOpenAndInit-table1-pool-0 {}] regionserver.HRegion(1676): Region close journal for ef8dfce57d5182c0203767dc7e68adf6: Waiting for close lock at 1731610595107Disabling compacts and flushes for region at 1731610595107Disabling writes for close at 1731610595108 (+1 ms)Writing region close event to WAL at 1731610595108Closed at 1731610595108 2024-11-14T18:56:35,110 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=table1 execute state=CREATE_TABLE_ADD_TO_META 2024-11-14T18:56:35,114 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"table1,,1731610595066.ef8dfce57d5182c0203767dc7e68adf6.","families":{"info":[{"qualifier":"regioninfo","vlen":40,"tag":[],"timestamp":"1731610595110"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731610595110"}]},"ts":"1731610595110"} 2024-11-14T18:56:35,119 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-14T18:56:35,120 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=table1 execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-14T18:56:35,124 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"table1","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731610595121"}]},"ts":"1731610595121"} 2024-11-14T18:56:35,131 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=table1, state=ENABLING in hbase:meta 2024-11-14T18:56:35,134 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=table1, region=ef8dfce57d5182c0203767dc7e68adf6, ASSIGN}] 2024-11-14T18:56:35,137 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=table1, region=ef8dfce57d5182c0203767dc7e68adf6, ASSIGN 2024-11-14T18:56:35,138 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=table1, region=ef8dfce57d5182c0203767dc7e68adf6, ASSIGN; state=OFFLINE, location=853c158ffd74,41231,1731610591853; forceNewPlan=false, retain=false 2024-11-14T18:56:35,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36053 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-14T18:56:35,290 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=ef8dfce57d5182c0203767dc7e68adf6, regionState=OPENING, regionLocation=853c158ffd74,41231,1731610591853 2024-11-14T18:56:35,293 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=table1, region=ef8dfce57d5182c0203767dc7e68adf6, ASSIGN because future has completed 2024-11-14T18:56:35,294 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure ef8dfce57d5182c0203767dc7e68adf6, server=853c158ffd74,41231,1731610591853}] 2024-11-14T18:56:35,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36053 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-14T18:56:35,450 INFO [RS_OPEN_REGION-regionserver/853c158ffd74:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(132): Open table1,,1731610595066.ef8dfce57d5182c0203767dc7e68adf6. 2024-11-14T18:56:35,450 DEBUG [RS_OPEN_REGION-regionserver/853c158ffd74:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7752): Opening region: {ENCODED => ef8dfce57d5182c0203767dc7e68adf6, NAME => 'table1,,1731610595066.ef8dfce57d5182c0203767dc7e68adf6.', STARTKEY => '', ENDKEY => ''} 2024-11-14T18:56:35,451 DEBUG [RS_OPEN_REGION-regionserver/853c158ffd74:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table table1 ef8dfce57d5182c0203767dc7e68adf6 2024-11-14T18:56:35,451 DEBUG [RS_OPEN_REGION-regionserver/853c158ffd74:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(898): Instantiated table1,,1731610595066.ef8dfce57d5182c0203767dc7e68adf6.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-14T18:56:35,451 DEBUG [RS_OPEN_REGION-regionserver/853c158ffd74:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7794): checking encryption for ef8dfce57d5182c0203767dc7e68adf6 2024-11-14T18:56:35,451 DEBUG [RS_OPEN_REGION-regionserver/853c158ffd74:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7797): checking classloading for ef8dfce57d5182c0203767dc7e68adf6 2024-11-14T18:56:35,453 INFO [StoreOpener-ef8dfce57d5182c0203767dc7e68adf6-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=true, for column family family of region ef8dfce57d5182c0203767dc7e68adf6 2024-11-14T18:56:35,457 INFO [StoreOpener-ef8dfce57d5182c0203767dc7e68adf6-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ef8dfce57d5182c0203767dc7e68adf6 columnFamilyName family 2024-11-14T18:56:35,457 DEBUG [StoreOpener-ef8dfce57d5182c0203767dc7e68adf6-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T18:56:35,458 INFO [StoreOpener-ef8dfce57d5182c0203767dc7e68adf6-1 {}] regionserver.HStore(327): Store=ef8dfce57d5182c0203767dc7e68adf6/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T18:56:35,458 DEBUG [RS_OPEN_REGION-regionserver/853c158ffd74:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1038): replaying wal for ef8dfce57d5182c0203767dc7e68adf6 2024-11-14T18:56:35,459 DEBUG [RS_OPEN_REGION-regionserver/853c158ffd74:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e5e0d570-5e1f-a378-f19c-4a0a374d5e72/b8642a29-d0d1-92cd-8fbf-cfee23d7ae40/data/default/table1/ef8dfce57d5182c0203767dc7e68adf6 2024-11-14T18:56:35,459 DEBUG [RS_OPEN_REGION-regionserver/853c158ffd74:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e5e0d570-5e1f-a378-f19c-4a0a374d5e72/b8642a29-d0d1-92cd-8fbf-cfee23d7ae40/data/default/table1/ef8dfce57d5182c0203767dc7e68adf6 2024-11-14T18:56:35,459 DEBUG [RS_OPEN_REGION-regionserver/853c158ffd74:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1048): stopping wal replay for ef8dfce57d5182c0203767dc7e68adf6 2024-11-14T18:56:35,459 DEBUG [RS_OPEN_REGION-regionserver/853c158ffd74:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1060): Cleaning up temporary data for ef8dfce57d5182c0203767dc7e68adf6 2024-11-14T18:56:35,460 DEBUG [RS_OPEN_REGION-regionserver/853c158ffd74:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1093): writing seq id for ef8dfce57d5182c0203767dc7e68adf6 2024-11-14T18:56:35,466 DEBUG [RS_OPEN_REGION-regionserver/853c158ffd74:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e5e0d570-5e1f-a378-f19c-4a0a374d5e72/b8642a29-d0d1-92cd-8fbf-cfee23d7ae40/data/default/table1/ef8dfce57d5182c0203767dc7e68adf6/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-14T18:56:35,466 INFO [RS_OPEN_REGION-regionserver/853c158ffd74:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1114): Opened ef8dfce57d5182c0203767dc7e68adf6; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=74625982, jitterRate=0.1120137870311737}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-14T18:56:35,467 DEBUG [RS_OPEN_REGION-regionserver/853c158ffd74:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1122): Running coprocessor post-open hooks for ef8dfce57d5182c0203767dc7e68adf6 2024-11-14T18:56:35,468 DEBUG [RS_OPEN_REGION-regionserver/853c158ffd74:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1006): Region open journal for ef8dfce57d5182c0203767dc7e68adf6: Running coprocessor pre-open hook at 1731610595451Writing region info on filesystem at 1731610595451Initializing all the Stores at 1731610595452 (+1 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731610595452Cleaning up temporary data from old regions at 1731610595459 (+7 ms)Running coprocessor post-open hooks at 1731610595467 (+8 ms)Region opened successfully at 1731610595467 2024-11-14T18:56:35,469 INFO [RS_OPEN_REGION-regionserver/853c158ffd74:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2236): Post open deploy tasks for table1,,1731610595066.ef8dfce57d5182c0203767dc7e68adf6., pid=6, masterSystemTime=1731610595447 2024-11-14T18:56:35,472 DEBUG [RS_OPEN_REGION-regionserver/853c158ffd74:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2266): Finished post open deploy task for table1,,1731610595066.ef8dfce57d5182c0203767dc7e68adf6. 2024-11-14T18:56:35,472 INFO [RS_OPEN_REGION-regionserver/853c158ffd74:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(153): Opened table1,,1731610595066.ef8dfce57d5182c0203767dc7e68adf6. 2024-11-14T18:56:35,472 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=ef8dfce57d5182c0203767dc7e68adf6, regionState=OPEN, openSeqNum=2, regionLocation=853c158ffd74,41231,1731610591853 2024-11-14T18:56:35,476 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure ef8dfce57d5182c0203767dc7e68adf6, server=853c158ffd74,41231,1731610591853 because future has completed 2024-11-14T18:56:35,479 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=6, resume processing ppid=5 2024-11-14T18:56:35,480 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure ef8dfce57d5182c0203767dc7e68adf6, server=853c158ffd74,41231,1731610591853 in 183 msec 2024-11-14T18:56:35,482 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=5, resume processing ppid=4 2024-11-14T18:56:35,482 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=table1, region=ef8dfce57d5182c0203767dc7e68adf6, ASSIGN in 345 msec 2024-11-14T18:56:35,483 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=table1 execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-14T18:56:35,483 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"table1","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731610595483"}]},"ts":"1731610595483"} 2024-11-14T18:56:35,485 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=table1, state=ENABLED in hbase:meta 2024-11-14T18:56:35,486 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=table1 execute state=CREATE_TABLE_POST_OPERATION 2024-11-14T18:56:35,489 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=table1 in 413 msec 2024-11-14T18:56:35,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36053 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-14T18:56:35,717 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table table1 get assigned. Timeout = 60000ms 2024-11-14T18:56:35,717 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:table1 completed 2024-11-14T18:56:35,720 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-14T18:56:35,728 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table table1 assigned to meta. Checking AM states. 2024-11-14T18:56:35,728 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-14T18:56:35,729 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table table1 assigned. 2024-11-14T18:56:35,740 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'table1', row='row1', locateType=CURRENT is [region=table1,,1731610595066.ef8dfce57d5182c0203767dc7e68adf6., hostname=853c158ffd74,41231,1731610591853, seqNum=2] 2024-11-14T18:56:35,754 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing ef8dfce57d5182c0203767dc7e68adf6 1/1 column families, dataSize=78 B heapSize=480 B 2024-11-14T18:56:35,785 DEBUG [Time-limited test {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 6b5d67bc0fbf44ada8ce04ab4d263ce9 2024-11-14T18:56:35,825 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e5e0d570-5e1f-a378-f19c-4a0a374d5e72/b8642a29-d0d1-92cd-8fbf-cfee23d7ae40/data/default/table1/ef8dfce57d5182c0203767dc7e68adf6/.tmp/family/6b5d67bc0fbf44ada8ce04ab4d263ce9 is 43, key is row1/family:qf1/1/Put/seqid=0 2024-11-14T18:56:35,830 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=78 B at sequenceid=6 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e5e0d570-5e1f-a378-f19c-4a0a374d5e72/b8642a29-d0d1-92cd-8fbf-cfee23d7ae40/data/default/table1/ef8dfce57d5182c0203767dc7e68adf6/.tmp/family/6b5d67bc0fbf44ada8ce04ab4d263ce9 2024-11-14T18:56:35,868 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e5e0d570-5e1f-a378-f19c-4a0a374d5e72/b8642a29-d0d1-92cd-8fbf-cfee23d7ae40/data/default/table1/ef8dfce57d5182c0203767dc7e68adf6/.tmp/family/6b5d67bc0fbf44ada8ce04ab4d263ce9 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e5e0d570-5e1f-a378-f19c-4a0a374d5e72/b8642a29-d0d1-92cd-8fbf-cfee23d7ae40/data/default/table1/ef8dfce57d5182c0203767dc7e68adf6/family/6b5d67bc0fbf44ada8ce04ab4d263ce9 2024-11-14T18:56:35,869 DEBUG [Time-limited test {}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e5e0d570-5e1f-a378-f19c-4a0a374d5e72/b8642a29-d0d1-92cd-8fbf-cfee23d7ae40/data/default/table1/ef8dfce57d5182c0203767dc7e68adf6/family/6b5d67bc0fbf44ada8ce04ab4d263ce9, delay=996 ms 2024-11-14T18:56:35,871 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e5e0d570-5e1f-a378-f19c-4a0a374d5e72/b8642a29-d0d1-92cd-8fbf-cfee23d7ae40/data/default/table1/ef8dfce57d5182c0203767dc7e68adf6/family/6b5d67bc0fbf44ada8ce04ab4d263ce9, entries=2, sequenceid=6, filesize=4.9 K 2024-11-14T18:56:35,877 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~78 B/78, heapSize ~464 B/464, currentSize=0 B/0 for ef8dfce57d5182c0203767dc7e68adf6 in 124ms, sequenceid=6, compaction requested=false 2024-11-14T18:56:35,878 DEBUG [Time-limited test {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'table1' 2024-11-14T18:56:35,880 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for ef8dfce57d5182c0203767dc7e68adf6: 2024-11-14T18:56:36,867 DEBUG [hfile-prefetch-1731610595869 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 6b5d67bc0fbf44ada8ce04ab4d263ce9_0. 2024-11-14T18:56:36,872 DEBUG [hfile-prefetch-1731610595869 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 6b5d67bc0fbf44ada8ce04ab4d263ce9_117. 2024-11-14T18:56:36,873 DEBUG [hfile-prefetch-1731610595869 {}] bucket.BucketCache(2282): Notifying caching completed for file file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e5e0d570-5e1f-a378-f19c-4a0a374d5e72/b8642a29-d0d1-92cd-8fbf-cfee23d7ae40/data/default/table1/ef8dfce57d5182c0203767dc7e68adf6/family/6b5d67bc0fbf44ada8ce04ab4d263ce9, with total blocks 2, and data blocks 1 2024-11-14T18:56:36,873 DEBUG [hfile-prefetch-1731610595869 {}] bucket.BucketCache(2286): iterating over 0 entries in the backing map 2024-11-14T18:56:36,873 DEBUG [hfile-prefetch-1731610595869 {}] bucket.BucketCache(2309): Prefetch executor completed for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e5e0d570-5e1f-a378-f19c-4a0a374d5e72/b8642a29-d0d1-92cd-8fbf-cfee23d7ae40/data/default/table1/ef8dfce57d5182c0203767dc7e68adf6/family/6b5d67bc0fbf44ada8ce04ab4d263ce9, but only 0 data blocks were cached. Total data blocks for file: 1. Checking for blocks pending cache in cache writer queue. 2024-11-14T18:56:36,874 DEBUG [hfile-prefetch-1731610595869 {}] bucket.BucketCache(2319): There are still blocks pending caching for file file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e5e0d570-5e1f-a378-f19c-4a0a374d5e72/b8642a29-d0d1-92cd-8fbf-cfee23d7ae40/data/default/table1/ef8dfce57d5182c0203767dc7e68adf6/family/6b5d67bc0fbf44ada8ce04ab4d263ce9. Will sleep 100ms and try the verification again. 2024-11-14T18:56:36,974 DEBUG [hfile-prefetch-1731610595869 {}] bucket.BucketCache(2282): Notifying caching completed for file file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e5e0d570-5e1f-a378-f19c-4a0a374d5e72/b8642a29-d0d1-92cd-8fbf-cfee23d7ae40/data/default/table1/ef8dfce57d5182c0203767dc7e68adf6/family/6b5d67bc0fbf44ada8ce04ab4d263ce9, with total blocks 2, and data blocks 1 2024-11-14T18:56:36,974 DEBUG [hfile-prefetch-1731610595869 {}] bucket.BucketCache(2286): iterating over 1 entries in the backing map 2024-11-14T18:56:36,974 DEBUG [hfile-prefetch-1731610595869 {}] bucket.BucketCache(2294): found block for file 6b5d67bc0fbf44ada8ce04ab4d263ce9 in the backing map. Acquiring read lock for offset 0 2024-11-14T18:56:36,974 DEBUG [hfile-prefetch-1731610595869 {}] bucket.BucketCache(2306): File file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e5e0d570-5e1f-a378-f19c-4a0a374d5e72/b8642a29-d0d1-92cd-8fbf-cfee23d7ae40/data/default/table1/ef8dfce57d5182c0203767dc7e68adf6/family/6b5d67bc0fbf44ada8ce04ab4d263ce9 has now been fully cached. 2024-11-14T18:56:36,975 DEBUG [hfile-prefetch-1731610595869 {}] hfile.PrefetchExecutor(121): Prefetch completed for 6b5d67bc0fbf44ada8ce04ab4d263ce9 2024-11-14T18:56:36,984 DEBUG [bucket-cache-persister {}] bucket.BucketCachePersister(47): Cache is inconsistent, persisting to disk 2024-11-14T18:56:36,984 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1488): Thread bucket-cache-persister started persisting bucket cache to file 2024-11-14T18:56:36,984 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1495): Persist in new chunked persistence format. 2024-11-14T18:56:36,984 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1674): persistToFile: before persisting backing map size: 1, fullycachedFiles size: 1, chunkSize: 10000 2024-11-14T18:56:37,006 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1681): persistToFile: after persisting backing map size: 1, fullycachedFiles size: 1 2024-11-14T18:56:37,006 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1499): PersistToFile: after persisting backing map size: 1, fullycachedFiles size: 1, file name: bucket.persistence1731610596984 2024-11-14T18:56:37,006 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1510): Thread bucket-cache-persister finished persisting bucket cache to file, renaming 2024-11-14T18:56:37,884 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-14T18:56:37,886 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37524, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-14T18:56:37,889 INFO [Time-limited test {}] hbase.SingleProcessHBaseCluster(504): Stopping Thread[RS:0;853c158ffd74:41231,5,FailOnTimeoutGroup] 2024-11-14T18:56:37,889 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '853c158ffd74,41231,1731610591853' ***** 2024-11-14T18:56:37,889 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Stopping rs 0 2024-11-14T18:56:37,890 INFO [Time-limited test {}] hfile.TestPrefetchRSClose(129): Stopped Region Server 0. 2024-11-14T18:56:37,890 INFO [RS:0;853c158ffd74:41231 {}] bucket.BucketCache(1739): Disabling cache 2024-11-14T18:56:37,890 INFO [RS:0;853c158ffd74:41231 {}] bucket.BucketCache(1763): Shutdown bucket cache: IO persistent=true; path to write=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e5e0d570-5e1f-a378-f19c-4a0a374d5e72/bucket.persistence 2024-11-14T18:56:37,890 INFO [Time-limited test-BucketCacheWriter-0 {}] bucket.BucketCache$WriterThread(1272): Time-limited test-BucketCacheWriter-0 exiting, cacheEnabled=false 2024-11-14T18:56:37,890 INFO [Time-limited test-BucketCacheWriter-2 {}] bucket.BucketCache$WriterThread(1272): Time-limited test-BucketCacheWriter-2 exiting, cacheEnabled=false 2024-11-14T18:56:37,890 INFO [Time-limited test-BucketCacheWriter-1 {}] bucket.BucketCache$WriterThread(1272): Time-limited test-BucketCacheWriter-1 exiting, cacheEnabled=false 2024-11-14T18:56:37,890 INFO [RS:0;853c158ffd74:41231 {}] bucket.BucketCache(1769): Shutting down cache persister thread. 2024-11-14T18:56:37,891 WARN [bucket-cache-persister {}] bucket.BucketCachePersister(64): Interrupting BucketCachePersister thread. java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCachePersister.run(BucketCachePersister.java:45) ~[classes/:?] 2024-11-14T18:56:37,901 DEBUG [RS:0;853c158ffd74:41231 {}] bucket.BucketCache(1488): Thread RS:0;853c158ffd74:41231 started persisting bucket cache to file 2024-11-14T18:56:37,901 DEBUG [RS:0;853c158ffd74:41231 {}] bucket.BucketCache(1495): Persist in new chunked persistence format. 2024-11-14T18:56:37,901 DEBUG [RS:0;853c158ffd74:41231 {}] bucket.BucketCache(1674): persistToFile: before persisting backing map size: 1, fullycachedFiles size: 1, chunkSize: 10000 2024-11-14T18:56:37,904 DEBUG [RS:0;853c158ffd74:41231 {}] bucket.BucketCache(1681): persistToFile: after persisting backing map size: 1, fullycachedFiles size: 1 2024-11-14T18:56:37,904 DEBUG [RS:0;853c158ffd74:41231 {}] bucket.BucketCache(1499): PersistToFile: after persisting backing map size: 1, fullycachedFiles size: 1, file name: bucket.persistence1731610597901 2024-11-14T18:56:37,904 DEBUG [RS:0;853c158ffd74:41231 {}] bucket.BucketCache(1510): Thread RS:0;853c158ffd74:41231 finished persisting bucket cache to file, renaming 2024-11-14T18:56:37,941 INFO [RS:0;853c158ffd74:41231 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-14T18:56:37,941 INFO [RS:0;853c158ffd74:41231 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-14T18:56:37,941 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-14T18:56:37,941 INFO [RS:0;853c158ffd74:41231 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-14T18:56:37,941 INFO [RS:0;853c158ffd74:41231 {}] regionserver.HRegionServer(3091): Received CLOSE for ef8dfce57d5182c0203767dc7e68adf6 2024-11-14T18:56:37,942 INFO [RS:0;853c158ffd74:41231 {}] regionserver.HRegionServer(959): stopping server 853c158ffd74,41231,1731610591853 2024-11-14T18:56:37,942 INFO [RS:0;853c158ffd74:41231 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-14T18:56:37,942 INFO [RS:0;853c158ffd74:41231 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;853c158ffd74:41231. 2024-11-14T18:56:37,942 INFO [RS_CLOSE_REGION-regionserver/853c158ffd74:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1662): Closing region table1,,1731610595066.ef8dfce57d5182c0203767dc7e68adf6. during a graceful stop, and cache persistence is on, so setting evict on close to false. 2024-11-14T18:56:37,943 DEBUG [RS:0;853c158ffd74:41231 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-14T18:56:37,943 DEBUG [RS_CLOSE_REGION-regionserver/853c158ffd74:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing ef8dfce57d5182c0203767dc7e68adf6, disabling compactions & flushes 2024-11-14T18:56:37,943 INFO [RS_CLOSE_REGION-regionserver/853c158ffd74:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region table1,,1731610595066.ef8dfce57d5182c0203767dc7e68adf6. 2024-11-14T18:56:37,943 DEBUG [RS_CLOSE_REGION-regionserver/853c158ffd74:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on table1,,1731610595066.ef8dfce57d5182c0203767dc7e68adf6. 2024-11-14T18:56:37,943 DEBUG [RS_CLOSE_REGION-regionserver/853c158ffd74:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on table1,,1731610595066.ef8dfce57d5182c0203767dc7e68adf6. after waiting 0 ms 2024-11-14T18:56:37,943 DEBUG [RS_CLOSE_REGION-regionserver/853c158ffd74:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region table1,,1731610595066.ef8dfce57d5182c0203767dc7e68adf6. 2024-11-14T18:56:37,948 DEBUG [RS:0;853c158ffd74:41231 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T18:56:37,949 INFO [RS:0;853c158ffd74:41231 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-14T18:56:37,949 INFO [RS:0;853c158ffd74:41231 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-14T18:56:37,949 INFO [RS:0;853c158ffd74:41231 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-14T18:56:37,949 INFO [RS:0;853c158ffd74:41231 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-11-14T18:56:37,950 INFO [RS:0;853c158ffd74:41231 {}] regionserver.HRegionServer(1321): Waiting on 2 regions to close 2024-11-14T18:56:37,950 INFO [RS_CLOSE_META-regionserver/853c158ffd74:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1662): Closing region hbase:meta,,1.1588230740 during a graceful stop, and cache persistence is on, so setting evict on close to false. 2024-11-14T18:56:37,950 DEBUG [RS:0;853c158ffd74:41231 {}] regionserver.HRegionServer(1325): Online Regions={ef8dfce57d5182c0203767dc7e68adf6=table1,,1731610595066.ef8dfce57d5182c0203767dc7e68adf6., 1588230740=hbase:meta,,1.1588230740} 2024-11-14T18:56:37,950 DEBUG [RS_CLOSE_REGION-regionserver/853c158ffd74:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e5e0d570-5e1f-a378-f19c-4a0a374d5e72/b8642a29-d0d1-92cd-8fbf-cfee23d7ae40/data/default/table1/ef8dfce57d5182c0203767dc7e68adf6/recovered.edits/9.seqid, newMaxSeqId=9, maxSeqId=1 2024-11-14T18:56:37,950 DEBUG [RS_CLOSE_META-regionserver/853c158ffd74:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-14T18:56:37,950 INFO [RS_CLOSE_META-regionserver/853c158ffd74:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-14T18:56:37,950 DEBUG [RS_CLOSE_META-regionserver/853c158ffd74:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-14T18:56:37,951 DEBUG [RS_CLOSE_META-regionserver/853c158ffd74:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-14T18:56:37,951 DEBUG [RS_CLOSE_META-regionserver/853c158ffd74:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-14T18:56:37,951 DEBUG [RS:0;853c158ffd74:41231 {}] regionserver.HRegionServer(1351): Waiting on 1588230740, ef8dfce57d5182c0203767dc7e68adf6 2024-11-14T18:56:37,951 INFO [RS_CLOSE_META-regionserver/853c158ffd74:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=1.20 KB heapSize=3.23 KB 2024-11-14T18:56:37,951 INFO [RS_CLOSE_REGION-regionserver/853c158ffd74:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed table1,,1731610595066.ef8dfce57d5182c0203767dc7e68adf6. 2024-11-14T18:56:37,952 DEBUG [RS_CLOSE_REGION-regionserver/853c158ffd74:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for ef8dfce57d5182c0203767dc7e68adf6: Waiting for close lock at 1731610597942Running coprocessor pre-close hooks at 1731610597943 (+1 ms)Disabling compacts and flushes for region at 1731610597943Disabling writes for close at 1731610597943Writing region close event to WAL at 1731610597945 (+2 ms)Running coprocessor post-close hooks at 1731610597950 (+5 ms)Closed at 1731610597951 (+1 ms) 2024-11-14T18:56:37,952 DEBUG [RS_CLOSE_REGION-regionserver/853c158ffd74:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed table1,,1731610595066.ef8dfce57d5182c0203767dc7e68adf6. 2024-11-14T18:56:37,962 INFO [regionserver/853c158ffd74:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-14T18:56:37,962 DEBUG [RS_CLOSE_META-regionserver/853c158ffd74:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFile$WriterFactory(308): Unable to set drop behind on ae532d8626b342bfade7c95e96ae9ee6 2024-11-14T18:56:37,978 DEBUG [RS_CLOSE_META-regionserver/853c158ffd74:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e5e0d570-5e1f-a378-f19c-4a0a374d5e72/b8642a29-d0d1-92cd-8fbf-cfee23d7ae40/data/hbase/meta/1588230740/.tmp/info/ae532d8626b342bfade7c95e96ae9ee6 is 133, key is table1,,1731610595066.ef8dfce57d5182c0203767dc7e68adf6./info:regioninfo/1731610595472/Put/seqid=0 2024-11-14T18:56:37,979 INFO [RS_CLOSE_META-regionserver/853c158ffd74:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.05 KB at sequenceid=11 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e5e0d570-5e1f-a378-f19c-4a0a374d5e72/b8642a29-d0d1-92cd-8fbf-cfee23d7ae40/data/hbase/meta/1588230740/.tmp/info/ae532d8626b342bfade7c95e96ae9ee6 2024-11-14T18:56:37,987 DEBUG [RS_CLOSE_META-regionserver/853c158ffd74:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 5c8725fef4be4317ab1d0ba0e2171e25 2024-11-14T18:56:38,005 DEBUG [RS_CLOSE_META-regionserver/853c158ffd74:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e5e0d570-5e1f-a378-f19c-4a0a374d5e72/b8642a29-d0d1-92cd-8fbf-cfee23d7ae40/data/hbase/meta/1588230740/.tmp/ns/5c8725fef4be4317ab1d0ba0e2171e25 is 43, key is default/ns:d/1731610594851/Put/seqid=0 2024-11-14T18:56:38,006 INFO [RS_CLOSE_META-regionserver/853c158ffd74:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=74 B at sequenceid=11 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e5e0d570-5e1f-a378-f19c-4a0a374d5e72/b8642a29-d0d1-92cd-8fbf-cfee23d7ae40/data/hbase/meta/1588230740/.tmp/ns/5c8725fef4be4317ab1d0ba0e2171e25 2024-11-14T18:56:38,014 DEBUG [RS_CLOSE_META-regionserver/853c158ffd74:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFile$WriterFactory(308): Unable to set drop behind on de6f74621b704ac7b26af237ac6fd6d9 2024-11-14T18:56:38,027 DEBUG [RS_CLOSE_META-regionserver/853c158ffd74:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e5e0d570-5e1f-a378-f19c-4a0a374d5e72/b8642a29-d0d1-92cd-8fbf-cfee23d7ae40/data/hbase/meta/1588230740/.tmp/table/de6f74621b704ac7b26af237ac6fd6d9 is 42, key is table1/table:state/1731610595483/Put/seqid=0 2024-11-14T18:56:38,027 INFO [RS_CLOSE_META-regionserver/853c158ffd74:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=76 B at sequenceid=11 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e5e0d570-5e1f-a378-f19c-4a0a374d5e72/b8642a29-d0d1-92cd-8fbf-cfee23d7ae40/data/hbase/meta/1588230740/.tmp/table/de6f74621b704ac7b26af237ac6fd6d9 2024-11-14T18:56:38,030 DEBUG [RS_CLOSE_META-regionserver/853c158ffd74:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e5e0d570-5e1f-a378-f19c-4a0a374d5e72/b8642a29-d0d1-92cd-8fbf-cfee23d7ae40/data/hbase/meta/1588230740/.tmp/info/ae532d8626b342bfade7c95e96ae9ee6 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e5e0d570-5e1f-a378-f19c-4a0a374d5e72/b8642a29-d0d1-92cd-8fbf-cfee23d7ae40/data/hbase/meta/1588230740/info/ae532d8626b342bfade7c95e96ae9ee6 2024-11-14T18:56:38,032 DEBUG [RS_CLOSE_META-regionserver/853c158ffd74:0-0 {event_type=M_RS_CLOSE_META}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e5e0d570-5e1f-a378-f19c-4a0a374d5e72/b8642a29-d0d1-92cd-8fbf-cfee23d7ae40/data/hbase/meta/1588230740/info/ae532d8626b342bfade7c95e96ae9ee6, delay=953 ms 2024-11-14T18:56:38,034 INFO [RS_CLOSE_META-regionserver/853c158ffd74:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e5e0d570-5e1f-a378-f19c-4a0a374d5e72/b8642a29-d0d1-92cd-8fbf-cfee23d7ae40/data/hbase/meta/1588230740/info/ae532d8626b342bfade7c95e96ae9ee6, entries=10, sequenceid=11, filesize=6.3 K 2024-11-14T18:56:38,034 DEBUG [RS_CLOSE_META-regionserver/853c158ffd74:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e5e0d570-5e1f-a378-f19c-4a0a374d5e72/b8642a29-d0d1-92cd-8fbf-cfee23d7ae40/data/hbase/meta/1588230740/.tmp/ns/5c8725fef4be4317ab1d0ba0e2171e25 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e5e0d570-5e1f-a378-f19c-4a0a374d5e72/b8642a29-d0d1-92cd-8fbf-cfee23d7ae40/data/hbase/meta/1588230740/ns/5c8725fef4be4317ab1d0ba0e2171e25 2024-11-14T18:56:38,036 DEBUG [RS_CLOSE_META-regionserver/853c158ffd74:0-0 {event_type=M_RS_CLOSE_META}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e5e0d570-5e1f-a378-f19c-4a0a374d5e72/b8642a29-d0d1-92cd-8fbf-cfee23d7ae40/data/hbase/meta/1588230740/ns/5c8725fef4be4317ab1d0ba0e2171e25, delay=971 ms 2024-11-14T18:56:38,038 INFO [RS_CLOSE_META-regionserver/853c158ffd74:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e5e0d570-5e1f-a378-f19c-4a0a374d5e72/b8642a29-d0d1-92cd-8fbf-cfee23d7ae40/data/hbase/meta/1588230740/ns/5c8725fef4be4317ab1d0ba0e2171e25, entries=2, sequenceid=11, filesize=5.0 K 2024-11-14T18:56:38,038 DEBUG [RS_CLOSE_META-regionserver/853c158ffd74:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e5e0d570-5e1f-a378-f19c-4a0a374d5e72/b8642a29-d0d1-92cd-8fbf-cfee23d7ae40/data/hbase/meta/1588230740/.tmp/table/de6f74621b704ac7b26af237ac6fd6d9 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e5e0d570-5e1f-a378-f19c-4a0a374d5e72/b8642a29-d0d1-92cd-8fbf-cfee23d7ae40/data/hbase/meta/1588230740/table/de6f74621b704ac7b26af237ac6fd6d9 2024-11-14T18:56:38,040 DEBUG [RS_CLOSE_META-regionserver/853c158ffd74:0-0 {event_type=M_RS_CLOSE_META}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e5e0d570-5e1f-a378-f19c-4a0a374d5e72/b8642a29-d0d1-92cd-8fbf-cfee23d7ae40/data/hbase/meta/1588230740/table/de6f74621b704ac7b26af237ac6fd6d9, delay=993 ms 2024-11-14T18:56:38,041 INFO [RS_CLOSE_META-regionserver/853c158ffd74:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e5e0d570-5e1f-a378-f19c-4a0a374d5e72/b8642a29-d0d1-92cd-8fbf-cfee23d7ae40/data/hbase/meta/1588230740/table/de6f74621b704ac7b26af237ac6fd6d9, entries=2, sequenceid=11, filesize=5.1 K 2024-11-14T18:56:38,043 INFO [RS_CLOSE_META-regionserver/853c158ffd74:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~1.20 KB/1227, heapSize ~2.94 KB/3008, currentSize=0 B/0 for 1588230740 in 91ms, sequenceid=11, compaction requested=false 2024-11-14T18:56:38,043 DEBUG [RS_CLOSE_META-regionserver/853c158ffd74:0-0 {event_type=M_RS_CLOSE_META}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta' 2024-11-14T18:56:38,045 DEBUG [StoreFileCloser-1588230740-info-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e5e0d570-5e1f-a378-f19c-4a0a374d5e72/b8642a29-d0d1-92cd-8fbf-cfee23d7ae40/data/hbase/meta/1588230740/info/ae532d8626b342bfade7c95e96ae9ee6 2024-11-14T18:56:38,046 DEBUG [StoreFileCloser-1588230740-ns-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e5e0d570-5e1f-a378-f19c-4a0a374d5e72/b8642a29-d0d1-92cd-8fbf-cfee23d7ae40/data/hbase/meta/1588230740/ns/5c8725fef4be4317ab1d0ba0e2171e25 2024-11-14T18:56:38,046 DEBUG [StoreFileCloser-1588230740-table-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e5e0d570-5e1f-a378-f19c-4a0a374d5e72/b8642a29-d0d1-92cd-8fbf-cfee23d7ae40/data/hbase/meta/1588230740/table/de6f74621b704ac7b26af237ac6fd6d9 2024-11-14T18:56:38,051 DEBUG [RS_CLOSE_META-regionserver/853c158ffd74:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e5e0d570-5e1f-a378-f19c-4a0a374d5e72/b8642a29-d0d1-92cd-8fbf-cfee23d7ae40/data/hbase/meta/1588230740/recovered.edits/14.seqid, newMaxSeqId=14, maxSeqId=1 2024-11-14T18:56:38,051 DEBUG [RS_CLOSE_META-regionserver/853c158ffd74:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-14T18:56:38,051 INFO [RS_CLOSE_META-regionserver/853c158ffd74:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-14T18:56:38,052 DEBUG [RS_CLOSE_META-regionserver/853c158ffd74:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1731610597950Running coprocessor pre-close hooks at 1731610597950Disabling compacts and flushes for region at 1731610597950Disabling writes for close at 1731610597951 (+1 ms)Obtaining lock to block concurrent updates at 1731610597951Preparing flush snapshotting stores in 1588230740 at 1731610597951Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=1227, getHeapSize=3248, getOffHeapSize=0, getCellsCount=14 at 1731610597952 (+1 ms)Flushing stores of hbase:meta,,1.1588230740 at 1731610597952Flushing 1588230740/info: creating writer at 1731610597953 (+1 ms)Flushing 1588230740/info: appending metadata at 1731610597977 (+24 ms)Flushing 1588230740/info: closing flushed file at 1731610597977Flushing 1588230740/ns: creating writer at 1731610597981 (+4 ms)Flushing 1588230740/ns: appending metadata at 1731610598003 (+22 ms)Flushing 1588230740/ns: closing flushed file at 1731610598003Flushing 1588230740/table: creating writer at 1731610598009 (+6 ms)Flushing 1588230740/table: appending metadata at 1731610598026 (+17 ms)Flushing 1588230740/table: closing flushed file at 1731610598026Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@4b601c1d: reopening flushed file at 1731610598030 (+4 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@c7d3b64: reopening flushed file at 1731610598034 (+4 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@9de45ce: reopening flushed file at 1731610598038 (+4 ms)Finished flush of dataSize ~1.20 KB/1227, heapSize ~2.94 KB/3008, currentSize=0 B/0 for 1588230740 in 91ms, sequenceid=11, compaction requested=false at 1731610598043 (+5 ms)Writing region close event to WAL at 1731610598047 (+4 ms)Running coprocessor post-close hooks at 1731610598051 (+4 ms)Closed at 1731610598051 2024-11-14T18:56:38,052 DEBUG [RS_CLOSE_META-regionserver/853c158ffd74:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-11-14T18:56:38,137 INFO [regionserver/853c158ffd74:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: MemstoreFlusherChore was stopped 2024-11-14T18:56:38,137 INFO [regionserver/853c158ffd74:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: CompactionChecker was stopped 2024-11-14T18:56:38,151 INFO [RS:0;853c158ffd74:41231 {}] regionserver.HRegionServer(976): stopping server 853c158ffd74,41231,1731610591853; all regions closed. 2024-11-14T18:56:38,155 DEBUG [RS:0;853c158ffd74:41231 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e5e0d570-5e1f-a378-f19c-4a0a374d5e72/b8642a29-d0d1-92cd-8fbf-cfee23d7ae40/oldWALs 2024-11-14T18:56:38,156 INFO [RS:0;853c158ffd74:41231 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 853c158ffd74%2C41231%2C1731610591853.meta:.meta(num 1731610594650) 2024-11-14T18:56:38,157 DEBUG [RS:0;853c158ffd74:41231 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e5e0d570-5e1f-a378-f19c-4a0a374d5e72/b8642a29-d0d1-92cd-8fbf-cfee23d7ae40/oldWALs 2024-11-14T18:56:38,157 INFO [RS:0;853c158ffd74:41231 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 853c158ffd74%2C41231%2C1731610591853:(num 1731610594235) 2024-11-14T18:56:38,157 DEBUG [RS:0;853c158ffd74:41231 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T18:56:38,157 INFO [RS:0;853c158ffd74:41231 {}] regionserver.LeaseManager(133): Closed leases 2024-11-14T18:56:38,158 INFO [RS:0;853c158ffd74:41231 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-14T18:56:38,158 INFO [RS:0;853c158ffd74:41231 {}] hbase.ChoreService(370): Chore service for: regionserver/853c158ffd74:0 had [ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS] on shutdown 2024-11-14T18:56:38,158 INFO [RS:0;853c158ffd74:41231 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-14T18:56:38,158 INFO [regionserver/853c158ffd74:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-14T18:56:38,158 INFO [RS:0;853c158ffd74:41231 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:41231 2024-11-14T18:56:38,189 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41231-0x1013bfe4f770001, quorum=127.0.0.1:56960, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/853c158ffd74,41231,1731610591853 2024-11-14T18:56:38,189 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36053-0x1013bfe4f770000, quorum=127.0.0.1:56960, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-14T18:56:38,189 INFO [RS:0;853c158ffd74:41231 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-14T18:56:38,197 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [853c158ffd74,41231,1731610591853] 2024-11-14T18:56:38,205 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/853c158ffd74,41231,1731610591853 already deleted, retry=false 2024-11-14T18:56:38,206 INFO [RegionServerTracker-0 {}] master.ServerManager(695): Processing expiration of 853c158ffd74,41231,1731610591853 on 853c158ffd74,36053,1731610590719 2024-11-14T18:56:38,210 DEBUG [RegionServerTracker-0 {}] procedure2.ProcedureExecutor(1139): Stored pid=7, state=RUNNABLE:SERVER_CRASH_START, hasLock=false; ServerCrashProcedure 853c158ffd74,41231,1731610591853, splitWal=true, meta=true 2024-11-14T18:56:38,213 INFO [RegionServerTracker-0 {}] assignment.AssignmentManager(1991): Scheduled ServerCrashProcedure pid=7 for 853c158ffd74,41231,1731610591853 (carryingMeta=true) 853c158ffd74,41231,1731610591853/CRASHED/regionCount=2/lock=java.util.concurrent.locks.ReentrantReadWriteLock@77b2fee0[Write locks = 1, Read locks = 0], oldState=ONLINE. 2024-11-14T18:56:38,213 INFO [PEWorker-2 {}] procedure.ServerCrashProcedure(169): Start pid=7, state=RUNNABLE:SERVER_CRASH_START, hasLock=true; ServerCrashProcedure 853c158ffd74,41231,1731610591853, splitWal=true, meta=true 2024-11-14T18:56:38,215 INFO [PEWorker-2 {}] procedure.ServerCrashProcedure(339): Splitting WALs pid=7, state=RUNNABLE:SERVER_CRASH_SPLIT_META_LOGS, hasLock=true; ServerCrashProcedure 853c158ffd74,41231,1731610591853, splitWal=true, meta=true, isMeta: true 2024-11-14T18:56:38,216 DEBUG [PEWorker-2 {}] master.MasterWalManager(329): Renamed region directory: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e5e0d570-5e1f-a378-f19c-4a0a374d5e72/b8642a29-d0d1-92cd-8fbf-cfee23d7ae40/WALs/853c158ffd74,41231,1731610591853-splitting 2024-11-14T18:56:38,216 INFO [PEWorker-2 {}] master.SplitLogManager(171): file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e5e0d570-5e1f-a378-f19c-4a0a374d5e72/b8642a29-d0d1-92cd-8fbf-cfee23d7ae40/WALs/853c158ffd74,41231,1731610591853-splitting dir is empty, no logs to split. 2024-11-14T18:56:38,216 INFO [PEWorker-2 {}] master.SplitWALManager(105): 853c158ffd74,41231,1731610591853 WAL count=0, meta=true 2024-11-14T18:56:38,219 INFO [PEWorker-2 {}] master.SplitLogManager(171): file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e5e0d570-5e1f-a378-f19c-4a0a374d5e72/b8642a29-d0d1-92cd-8fbf-cfee23d7ae40/WALs/853c158ffd74,41231,1731610591853-splitting dir is empty, no logs to split. 2024-11-14T18:56:38,219 INFO [PEWorker-2 {}] master.SplitWALManager(105): 853c158ffd74,41231,1731610591853 WAL count=0, meta=true 2024-11-14T18:56:38,219 DEBUG [PEWorker-2 {}] procedure.ServerCrashProcedure(329): Check if 853c158ffd74,41231,1731610591853 WAL splitting is done? wals=0, meta=true 2024-11-14T18:56:38,221 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=8, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-11-14T18:56:38,223 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=8, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-14T18:56:38,223 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=8, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-11-14T18:56:38,298 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41231-0x1013bfe4f770001, quorum=127.0.0.1:56960, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-14T18:56:38,298 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41231-0x1013bfe4f770001, quorum=127.0.0.1:56960, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-14T18:56:38,298 INFO [RS:0;853c158ffd74:41231 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-14T18:56:38,298 INFO [RS:0;853c158ffd74:41231 {}] regionserver.HRegionServer(1031): Exiting; stopping=853c158ffd74,41231,1731610591853; zookeeper connection closed. 2024-11-14T18:56:38,299 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@13f7062b {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@13f7062b 2024-11-14T18:56:38,374 WARN [853c158ffd74:36053 {}] assignment.AssignmentManager(2443): No servers available; cannot place 1 unassigned regions. 2024-11-14T18:56:38,890 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster 2024-11-14T18:56:38,890 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-14T18:56:38,891 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79) at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611) at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.io.hfile.TestPrefetchRSClose.tearDown(TestPrefetchRSClose.java:136) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-14T18:56:38,891 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T18:56:38,891 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T18:56:38,891 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-14T18:56:38,892 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2024-11-14T18:56:38,892 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=1590087585, stopped=false 2024-11-14T18:56:38,892 INFO [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=853c158ffd74,36053,1731610590719 2024-11-14T18:56:38,892 INFO [Time-limited test {}] master.HMaster(3321): ***** STOPPING master '853c158ffd74,36053,1731610590719' ***** 2024-11-14T18:56:38,892 INFO [Time-limited test {}] master.HMaster(3323): STOPPED: OnlineServer=0 right after cluster shutdown set 2024-11-14T18:56:38,893 INFO [M:0;853c158ffd74:36053 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-14T18:56:38,893 INFO [M:0;853c158ffd74:36053 {}] client.AsyncConnectionImpl(233): Connection has been closed by M:0;853c158ffd74:36053. 2024-11-14T18:56:38,893 DEBUG [M:0;853c158ffd74:36053 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.master.HMaster.run(HMaster.java:630) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-14T18:56:38,893 DEBUG [M:0;853c158ffd74:36053 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T18:56:38,893 INFO [M:0;853c158ffd74:36053 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-14T18:56:38,893 DEBUG [M:0;853c158ffd74:36053 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2024-11-14T18:56:38,893 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2024-11-14T18:56:38,893 DEBUG [M:0;853c158ffd74:36053 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2024-11-14T18:56:38,893 DEBUG [master/853c158ffd74:0:becomeActiveMaster-HFileCleaner.small.0-1731610593854 {}] cleaner.HFileCleaner(306): Exit Thread[master/853c158ffd74:0:becomeActiveMaster-HFileCleaner.small.0-1731610593854,5,FailOnTimeoutGroup] 2024-11-14T18:56:38,893 DEBUG [master/853c158ffd74:0:becomeActiveMaster-HFileCleaner.large.0-1731610593853 {}] cleaner.HFileCleaner(306): Exit Thread[master/853c158ffd74:0:becomeActiveMaster-HFileCleaner.large.0-1731610593853,5,FailOnTimeoutGroup] 2024-11-14T18:56:38,893 INFO [M:0;853c158ffd74:36053 {}] hbase.ChoreService(370): Chore service for: master/853c158ffd74:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown 2024-11-14T18:56:38,894 INFO [M:0;853c158ffd74:36053 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-14T18:56:38,894 DEBUG [M:0;853c158ffd74:36053 {}] master.HMaster(1795): Stopping service threads 2024-11-14T18:56:38,894 INFO [M:0;853c158ffd74:36053 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2024-11-14T18:56:38,894 INFO [M:0;853c158ffd74:36053 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-14T18:56:38,896 INFO [M:0;853c158ffd74:36053 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-14T18:56:38,896 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2024-11-14T18:56:38,910 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36053-0x1013bfe4f770000, quorum=127.0.0.1:56960, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-11-14T18:56:38,910 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36053-0x1013bfe4f770000, quorum=127.0.0.1:56960, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-14T18:56:38,911 DEBUG [M:0;853c158ffd74:36053 {}] zookeeper.ZKUtil(347): master:36053-0x1013bfe4f770000, quorum=127.0.0.1:56960, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error) 2024-11-14T18:56:38,911 WARN [M:0;853c158ffd74:36053 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null 2024-11-14T18:56:38,911 INFO [M:0;853c158ffd74:36053 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e5e0d570-5e1f-a378-f19c-4a0a374d5e72/b8642a29-d0d1-92cd-8fbf-cfee23d7ae40/.lastflushedseqids 2024-11-14T18:56:38,921 INFO [M:0;853c158ffd74:36053 {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-14T18:56:38,922 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36053-0x1013bfe4f770000, quorum=127.0.0.1:56960, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-14T18:56:38,922 INFO [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 1 regionserver(s) complete 2024-11-14T18:56:38,923 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:36053-0x1013bfe4f770000, quorum=127.0.0.1:56960, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-14T18:56:39,124 DEBUG [853c158ffd74:36053 {}] assignment.AssignmentManager(2447): Stopped! Dropping assign of 1 queued regions. 2024-11-14T18:56:39,125 INFO [M:0;853c158ffd74:36053 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2024-11-14T18:56:39,125 DEBUG [M:0;853c158ffd74:36053 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-14T18:56:39,125 INFO [M:0;853c158ffd74:36053 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-14T18:56:39,125 DEBUG [M:0;853c158ffd74:36053 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-14T18:56:39,125 DEBUG [M:0;853c158ffd74:36053 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-14T18:56:39,125 DEBUG [M:0;853c158ffd74:36053 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-14T18:56:39,125 INFO [M:0;853c158ffd74:36053 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=24.17 KB heapSize=30.80 KB 2024-11-14T18:56:39,134 DEBUG [M:0;853c158ffd74:36053 {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 9c07f2d6a8784b8a926f1fa4198f7ad7 2024-11-14T18:56:39,145 DEBUG [M:0;853c158ffd74:36053 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e5e0d570-5e1f-a378-f19c-4a0a374d5e72/b8642a29-d0d1-92cd-8fbf-cfee23d7ae40/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/9c07f2d6a8784b8a926f1fa4198f7ad7 is 82, key is hbase:meta,,1/info:regioninfo/1731610594728/Put/seqid=0 2024-11-14T18:56:39,146 INFO [M:0;853c158ffd74:36053 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=504 B at sequenceid=64 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e5e0d570-5e1f-a378-f19c-4a0a374d5e72/b8642a29-d0d1-92cd-8fbf-cfee23d7ae40/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/9c07f2d6a8784b8a926f1fa4198f7ad7 2024-11-14T18:56:39,153 DEBUG [M:0;853c158ffd74:36053 {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 351d6973b7b84cd6ac64cf0d718b6710 2024-11-14T18:56:39,165 DEBUG [M:0;853c158ffd74:36053 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e5e0d570-5e1f-a378-f19c-4a0a374d5e72/b8642a29-d0d1-92cd-8fbf-cfee23d7ae40/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/351d6973b7b84cd6ac64cf0d718b6710 is 706, key is \x00\x00\x00\x00\x00\x00\x00\x04/proc:d/1731610595488/Put/seqid=0 2024-11-14T18:56:39,166 INFO [M:0;853c158ffd74:36053 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=23.56 KB at sequenceid=64 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e5e0d570-5e1f-a378-f19c-4a0a374d5e72/b8642a29-d0d1-92cd-8fbf-cfee23d7ae40/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/351d6973b7b84cd6ac64cf0d718b6710 2024-11-14T18:56:39,173 DEBUG [M:0;853c158ffd74:36053 {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 40440a998f924ad896cbab5b218f602a 2024-11-14T18:56:39,185 DEBUG [M:0;853c158ffd74:36053 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e5e0d570-5e1f-a378-f19c-4a0a374d5e72/b8642a29-d0d1-92cd-8fbf-cfee23d7ae40/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/40440a998f924ad896cbab5b218f602a is 58, key is 853c158ffd74,41231,1731610591853/rs:/1731610598215/DeleteFamily/seqid=0 2024-11-14T18:56:39,186 INFO [M:0;853c158ffd74:36053 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=119 B at sequenceid=64 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e5e0d570-5e1f-a378-f19c-4a0a374d5e72/b8642a29-d0d1-92cd-8fbf-cfee23d7ae40/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/40440a998f924ad896cbab5b218f602a 2024-11-14T18:56:39,188 INFO [M:0;853c158ffd74:36053 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 40440a998f924ad896cbab5b218f602a 2024-11-14T18:56:39,188 DEBUG [M:0;853c158ffd74:36053 {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e5e0d570-5e1f-a378-f19c-4a0a374d5e72/b8642a29-d0d1-92cd-8fbf-cfee23d7ae40/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/9c07f2d6a8784b8a926f1fa4198f7ad7 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e5e0d570-5e1f-a378-f19c-4a0a374d5e72/b8642a29-d0d1-92cd-8fbf-cfee23d7ae40/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/9c07f2d6a8784b8a926f1fa4198f7ad7 2024-11-14T18:56:39,189 INFO [M:0;853c158ffd74:36053 {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e5e0d570-5e1f-a378-f19c-4a0a374d5e72/b8642a29-d0d1-92cd-8fbf-cfee23d7ae40/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/9c07f2d6a8784b8a926f1fa4198f7ad7, entries=8, sequenceid=64, filesize=5.5 K 2024-11-14T18:56:39,190 DEBUG [M:0;853c158ffd74:36053 {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e5e0d570-5e1f-a378-f19c-4a0a374d5e72/b8642a29-d0d1-92cd-8fbf-cfee23d7ae40/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/351d6973b7b84cd6ac64cf0d718b6710 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e5e0d570-5e1f-a378-f19c-4a0a374d5e72/b8642a29-d0d1-92cd-8fbf-cfee23d7ae40/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/351d6973b7b84cd6ac64cf0d718b6710 2024-11-14T18:56:39,191 INFO [M:0;853c158ffd74:36053 {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e5e0d570-5e1f-a378-f19c-4a0a374d5e72/b8642a29-d0d1-92cd-8fbf-cfee23d7ae40/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/351d6973b7b84cd6ac64cf0d718b6710, entries=8, sequenceid=64, filesize=6.5 K 2024-11-14T18:56:39,191 DEBUG [M:0;853c158ffd74:36053 {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e5e0d570-5e1f-a378-f19c-4a0a374d5e72/b8642a29-d0d1-92cd-8fbf-cfee23d7ae40/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/40440a998f924ad896cbab5b218f602a as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e5e0d570-5e1f-a378-f19c-4a0a374d5e72/b8642a29-d0d1-92cd-8fbf-cfee23d7ae40/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/40440a998f924ad896cbab5b218f602a 2024-11-14T18:56:39,193 INFO [M:0;853c158ffd74:36053 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 40440a998f924ad896cbab5b218f602a 2024-11-14T18:56:39,193 INFO [M:0;853c158ffd74:36053 {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e5e0d570-5e1f-a378-f19c-4a0a374d5e72/b8642a29-d0d1-92cd-8fbf-cfee23d7ae40/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/40440a998f924ad896cbab5b218f602a, entries=1, sequenceid=64, filesize=5.2 K 2024-11-14T18:56:39,194 INFO [M:0;853c158ffd74:36053 {}] regionserver.HRegion(3140): Finished flush of dataSize ~24.17 KB/24747, heapSize ~30.51 KB/31240, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 69ms, sequenceid=64, compaction requested=false 2024-11-14T18:56:39,196 INFO [M:0;853c158ffd74:36053 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-14T18:56:39,196 DEBUG [M:0;853c158ffd74:36053 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1731610599125Disabling compacts and flushes for region at 1731610599125Disabling writes for close at 1731610599125Obtaining lock to block concurrent updates at 1731610599125Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1731610599125Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=24747, getHeapSize=31480, getOffHeapSize=0, getCellsCount=76 at 1731610599126 (+1 ms)Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1731610599126Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1731610599126Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1731610599145 (+19 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1731610599145Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1731610599148 (+3 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1731610599165 (+17 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1731610599165Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1731610599168 (+3 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1731610599184 (+16 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1731610599184Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@18c32b78: reopening flushed file at 1731610599188 (+4 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@209bb95a: reopening flushed file at 1731610599190 (+2 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@144d4a34: reopening flushed file at 1731610599191 (+1 ms)Finished flush of dataSize ~24.17 KB/24747, heapSize ~30.51 KB/31240, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 69ms, sequenceid=64, compaction requested=false at 1731610599194 (+3 ms)Writing region close event to WAL at 1731610599196 (+2 ms)Closed at 1731610599196 2024-11-14T18:56:39,199 INFO [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-14T18:56:39,199 INFO [M:0;853c158ffd74:36053 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down. 2024-11-14T18:56:39,199 INFO [M:0;853c158ffd74:36053 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:36053 2024-11-14T18:56:39,200 INFO [M:0;853c158ffd74:36053 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-14T18:56:39,327 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36053-0x1013bfe4f770000, quorum=127.0.0.1:56960, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-14T18:56:39,327 INFO [M:0;853c158ffd74:36053 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-14T18:56:39,327 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36053-0x1013bfe4f770000, quorum=127.0.0.1:56960, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-14T18:56:39,332 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers 2024-11-14T18:56:39,360 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down 2024-11-14T18:56:39,361 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-14T18:56:39,368 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: io.hfile.TestPrefetchRSClose#testPrefetchPersistence Thread=45 (was 12) Potentially hanging thread: SnapshotHandlerChoreCleaner java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-3-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HMaster-EventLoopGroup-1-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Async-Client-Retry-Timer-pool-0 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.waitForNextTick(HashedWheelTimer.java:598) app//org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:494) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: weak-ref-cleaner-strictcontextstorage java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:155) java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:176) app//io.opentelemetry.context.StrictContextStorage$PendingScopes.run(StrictContextStorage.java:269) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: master/853c158ffd74:0:becomeActiveMaster-MemStoreChunkPool Statistics java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: hfile-prefetch-1731610598036 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1177) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Timer for 'HBase' metrics system java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.util.TimerThread.mainLoop(Timer.java:563) java.base@17.0.11/java.util.TimerThread.run(Timer.java:516) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-3-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ForkJoinPool.commonPool-worker-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.ForkJoinPool.awaitWork(ForkJoinPool.java:1724) java.base@17.0.11/java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1623) java.base@17.0.11/java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:165) Potentially hanging thread: HBase-Metrics2-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-4 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ForkJoinPool.commonPool-worker-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkUntil(LockSupport.java:410) java.base@17.0.11/java.util.concurrent.ForkJoinPool.awaitWork(ForkJoinPool.java:1726) java.base@17.0.11/java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1623) java.base@17.0.11/java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:165) Potentially hanging thread: HMaster-EventLoopGroup-1-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: SessionTracker java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.zookeeper.server.SessionTrackerImpl.run(SessionTrackerImpl.java:163) Potentially hanging thread: hfile-prefetch-1731610595869 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1177) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HMaster-EventLoopGroup-1-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-3-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: hfile-prefetch-1731610598040 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1177) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ForkJoinPool.commonPool-worker-3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.ForkJoinPool.awaitWork(ForkJoinPool.java:1724) java.base@17.0.11/java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1623) java.base@17.0.11/java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:165) Potentially hanging thread: org.apache.hadoop.fs.FileSystem$Statistics$StatisticsDataReferenceCleaner java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:155) java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:176) app//org.apache.hadoop.fs.FileSystem$Statistics$StatisticsDataReferenceCleaner.run(FileSystem.java:4171) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Monitor thread for TaskMonitor java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hbase.monitoring.TaskMonitor$MonitorRunnable.run(TaskMonitor.java:325) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-5 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: region-location-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1170) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Idle-Rpc-Conn-Sweeper-pool-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: regionserver/853c158ffd74:0.procedureResultReporter java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.RemoteProcedureResultReporter.run(RemoteProcedureResultReporter.java:75) Potentially hanging thread: Time-limited test.named-queue-events-pool-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) app//com.lmax.disruptor.BlockingWaitStrategy.waitFor(BlockingWaitStrategy.java:47) app//com.lmax.disruptor.ProcessingSequenceBarrier.waitFor(ProcessingSequenceBarrier.java:56) app//com.lmax.disruptor.BatchEventProcessor.processEvents(BatchEventProcessor.java:159) app//com.lmax.disruptor.BatchEventProcessor.run(BatchEventProcessor.java:125) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RpcClient-timer-pool-0 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.waitForNextTick(HashedWheelTimer.java:598) app//org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:494) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: hfile-prefetch-1731610598032 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1170) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: master/853c158ffd74:0:becomeActiveMaster-MemStoreChunkPool Statistics java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=337 (was 287) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=470 (was 485), ProcessCount=11 (was 11), AvailableMemoryMB=3641 (was 3957)