2024-12-06 01:45:34,901 main DEBUG Apache Log4j Core 2.17.2 initializing configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@15a34df2 2024-12-06 01:45:34,923 main DEBUG Took 0.011001 seconds to load 1 plugins from package org.apache.hadoop.hbase.logging 2024-12-06 01:45:34,923 main DEBUG PluginManager 'Core' found 129 plugins 2024-12-06 01:45:34,924 main DEBUG PluginManager 'Level' found 0 plugins 2024-12-06 01:45:34,925 main DEBUG PluginManager 'Lookup' found 16 plugins 2024-12-06 01:45:34,927 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-06 01:45:34,936 main DEBUG PluginManager 'TypeConverter' found 26 plugins 2024-12-06 01:45:34,948 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.util.MBeans", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-06 01:45:34,949 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-06 01:45:34,950 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.logging.TestJul2Slf4j", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-06 01:45:34,950 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-06 01:45:34,951 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.zookeeper", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-06 01:45:34,951 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-06 01:45:34,952 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSinkAdapter", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-06 01:45:34,952 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-06 01:45:34,953 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSystemImpl", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-06 01:45:34,953 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-06 01:45:34,954 main DEBUG LoggerConfig$Builder(additivity="false", level="WARN", levelAndRefs="null", name="org.apache.directory", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-06 01:45:34,954 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-06 01:45:34,954 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.ipc.FailedServers", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-06 01:45:34,955 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-06 01:45:34,955 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsConfig", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-06 01:45:34,955 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-06 01:45:34,956 main DEBUG LoggerConfig$Builder(additivity="null", level="INFO", levelAndRefs="null", name="org.apache.hadoop.hbase.ScheduledChore", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-06 01:45:34,956 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-06 01:45:34,956 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.regionserver.RSRpcServices", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-06 01:45:34,957 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-06 01:45:34,957 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-06 01:45:34,957 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-06 01:45:34,958 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-06 01:45:34,958 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-06 01:45:34,958 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hbase.thirdparty.io.netty.channel", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-06 01:45:34,959 main DEBUG Building Plugin[name=root, class=org.apache.logging.log4j.core.config.LoggerConfig$RootLogger]. 2024-12-06 01:45:34,960 main DEBUG LoggerConfig$RootLogger$Builder(additivity="null", level="null", levelAndRefs="INFO,Console", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-06 01:45:34,961 main DEBUG Building Plugin[name=loggers, class=org.apache.logging.log4j.core.config.LoggersPlugin]. 2024-12-06 01:45:34,963 main DEBUG createLoggers(={org.apache.hadoop.metrics2.util.MBeans, org.apache.hadoop.hbase.logging.TestJul2Slf4j, org.apache.zookeeper, org.apache.hadoop.metrics2.impl.MetricsSinkAdapter, org.apache.hadoop.metrics2.impl.MetricsSystemImpl, org.apache.directory, org.apache.hadoop.hbase.ipc.FailedServers, org.apache.hadoop.metrics2.impl.MetricsConfig, org.apache.hadoop.hbase.ScheduledChore, org.apache.hadoop.hbase.regionserver.RSRpcServices, org.apache.hadoop, org.apache.hadoop.hbase, org.apache.hbase.thirdparty.io.netty.channel, root}) 2024-12-06 01:45:34,963 main DEBUG Building Plugin[name=layout, class=org.apache.logging.log4j.core.layout.PatternLayout]. 2024-12-06 01:45:34,964 main DEBUG PatternLayout$Builder(pattern="%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n", PatternSelector=null, Configuration(PropertiesConfig), Replace=null, charset="null", alwaysWriteExceptions="null", disableAnsi="null", noConsoleNoAnsi="null", header="null", footer="null") 2024-12-06 01:45:34,965 main DEBUG PluginManager 'Converter' found 47 plugins 2024-12-06 01:45:34,973 main DEBUG Building Plugin[name=appender, class=org.apache.hadoop.hbase.logging.HBaseTestAppender]. 2024-12-06 01:45:34,976 main DEBUG HBaseTestAppender$Builder(target="SYSTEM_ERR", maxSize="1G", bufferedIo="null", bufferSize="null", immediateFlush="null", ignoreExceptions="null", PatternLayout(%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n), name="Console", Configuration(PropertiesConfig), Filter=null, ={}) 2024-12-06 01:45:34,977 main DEBUG Starting HBaseTestOutputStreamManager SYSTEM_ERR 2024-12-06 01:45:34,977 main DEBUG Building Plugin[name=appenders, class=org.apache.logging.log4j.core.config.AppendersPlugin]. 2024-12-06 01:45:34,978 main DEBUG createAppenders(={Console}) 2024-12-06 01:45:34,978 main DEBUG Configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@15a34df2 initialized 2024-12-06 01:45:34,979 main DEBUG Starting configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@15a34df2 2024-12-06 01:45:34,979 main DEBUG Started configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@15a34df2 OK. 2024-12-06 01:45:34,979 main DEBUG Shutting down OutputStreamManager SYSTEM_OUT.false.false-1 2024-12-06 01:45:34,980 main DEBUG OutputStream closed 2024-12-06 01:45:34,980 main DEBUG Shut down OutputStreamManager SYSTEM_OUT.false.false-1, all resources released: true 2024-12-06 01:45:34,980 main DEBUG Appender DefaultConsole-1 stopped with status true 2024-12-06 01:45:34,981 main DEBUG Stopped org.apache.logging.log4j.core.config.DefaultConfiguration@4f2b503c OK 2024-12-06 01:45:35,065 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6 2024-12-06 01:45:35,067 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=StatusLogger 2024-12-06 01:45:35,068 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=ContextSelector 2024-12-06 01:45:35,069 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name= 2024-12-06 01:45:35,069 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.directory 2024-12-06 01:45:35,069 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSinkAdapter 2024-12-06 01:45:35,070 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.zookeeper 2024-12-06 01:45:35,070 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.logging.TestJul2Slf4j 2024-12-06 01:45:35,070 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSystemImpl 2024-12-06 01:45:35,071 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.util.MBeans 2024-12-06 01:45:35,071 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase 2024-12-06 01:45:35,071 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop 2024-12-06 01:45:35,071 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ipc.FailedServers 2024-12-06 01:45:35,072 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.regionserver.RSRpcServices 2024-12-06 01:45:35,072 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsConfig 2024-12-06 01:45:35,072 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hbase.thirdparty.io.netty.channel 2024-12-06 01:45:35,073 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ScheduledChore 2024-12-06 01:45:35,073 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Appenders,name=Console 2024-12-06 01:45:35,076 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-12-06 01:45:35,076 main DEBUG Reconfiguration complete for context[name=1dbd16a6] at URI jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-logging/target/hbase-logging-4.0.0-alpha-1-SNAPSHOT-tests.jar!/log4j2.properties (org.apache.logging.log4j.core.LoggerContext@31c7528f) with optional ClassLoader: null 2024-12-06 01:45:35,076 main DEBUG Shutdown hook enabled. Registering a new one. 2024-12-06 01:45:35,077 main DEBUG LoggerContext[name=1dbd16a6, org.apache.logging.log4j.core.LoggerContext@31c7528f] started OK. 2024-12-06T01:45:35,091 INFO [main {}] hbase.HBaseClassTestRule(94): Test class org.apache.hadoop.hbase.io.hfile.TestBlockEvictionOnRegionMovement timeout: 13 mins 2024-12-06 01:45:35,094 main DEBUG AsyncLogger.ThreadNameStrategy=UNCACHED (user specified null, default is UNCACHED) 2024-12-06 01:45:35,094 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-12-06T01:45:35,341 DEBUG [main {}] hbase.HBaseTestingUtil(323): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/eff72b7a-3322-3fdb-88e1-7287cd98d0d4 2024-12-06T01:45:35,367 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: io.hfile.TestBlockEvictionOnRegionMovement#testBlockEvictionOnRegionMove Thread=12, OpenFileDescriptor=287, MaxFileDescriptor=1048576, SystemLoadAverage=578, ProcessCount=11, AvailableMemoryMB=10459 2024-12-06T01:45:35,472 WARN [Time-limited test {}] util.NativeCodeLoader(60): Unable to load native-hadoop library for your platform... using builtin-java classes where applicable 2024-12-06T01:45:35,600 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-12-06T01:45:35,621 INFO [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/eff72b7a-3322-3fdb-88e1-7287cd98d0d4/cluster_0a3580e8-bc34-f298-9ae2-a9f29f465309, deleteOnExit=true 2024-12-06T01:45:35,715 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/eff72b7a-3322-3fdb-88e1-7287cd98d0d4/cluster_0a3580e8-bc34-f298-9ae2-a9f29f465309/zookeeper_0, clientPort=54114, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/eff72b7a-3322-3fdb-88e1-7287cd98d0d4/cluster_0a3580e8-bc34-f298-9ae2-a9f29f465309/zookeeper_0/version-2, dataDirSize=457 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/eff72b7a-3322-3fdb-88e1-7287cd98d0d4/cluster_0a3580e8-bc34-f298-9ae2-a9f29f465309/zookeeper_0/version-2, dataLogSize=457 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0 2024-12-06T01:45:35,729 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=54114 2024-12-06T01:45:35,729 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-12-06T01:45:35,825 INFO [Time-limited test {}] util.FSUtils(489): Created version file at /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/eff72b7a-3322-3fdb-88e1-7287cd98d0d4/e93bd390-0caa-6c00-f03a-52237a196617 with version=8 2024-12-06T01:45:35,825 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1139): Setting hbase.fs.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/eff72b7a-3322-3fdb-88e1-7287cd98d0d4/e93bd390-0caa-6c00-f03a-52237a196617/hbase-staging 2024-12-06T01:45:35,932 DEBUG [Time-limited test {}] channel.MultithreadEventLoopGroup(44): -Dio.netty.eventLoopThreads: 16 2024-12-06T01:45:36,239 INFO [Time-limited test {}] client.ConnectionUtils(128): master/00c764f7c3aa:0 server-side Connection retries=45 2024-12-06T01:45:36,250 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-06T01:45:36,252 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-12-06T01:45:36,262 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-12-06T01:45:36,263 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-06T01:45:36,263 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-12-06T01:45:36,423 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService 2024-12-06T01:45:36,520 WARN [Time-limited test {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-12-06T01:45:36,566 INFO [Time-limited test {}] metrics.MetricRegistriesLoader(60): Loaded MetricRegistries class org.apache.hadoop.hbase.metrics.impl.MetricRegistriesImpl 2024-12-06T01:45:36,579 DEBUG [Time-limited test {}] util.ClassSize(228): Using Unsafe to estimate memory layout 2024-12-06T01:45:36,584 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-12-06T01:45:36,629 DEBUG [Time-limited test {}] channel.DefaultChannelId(84): -Dio.netty.processId: 14403 (auto-detected) 2024-12-06T01:45:36,630 DEBUG [Time-limited test {}] channel.DefaultChannelId(106): -Dio.netty.machineId: 02:42:ac:ff:fe:11:00:02 (auto-detected) 2024-12-06T01:45:36,661 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:40497 2024-12-06T01:45:36,686 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:40497 connecting to ZooKeeper ensemble=127.0.0.1:54114 2024-12-06T01:45:36,723 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:404970x0, quorum=127.0.0.1:54114, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-06T01:45:36,726 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:40497-0x100650089b40000 connected 2024-12-06T01:45:36,754 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-12-06T01:45:36,755 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-12-06T01:45:36,763 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:40497-0x100650089b40000, quorum=127.0.0.1:54114, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-06T01:45:36,767 INFO [Time-limited test {}] master.HMaster(525): hbase.rootdir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/eff72b7a-3322-3fdb-88e1-7287cd98d0d4/e93bd390-0caa-6c00-f03a-52237a196617, hbase.cluster.distributed=false 2024-12-06T01:45:36,794 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:40497-0x100650089b40000, quorum=127.0.0.1:54114, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-12-06T01:45:36,802 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=40497 2024-12-06T01:45:36,803 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=40497 2024-12-06T01:45:36,806 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=40497 2024-12-06T01:45:36,806 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=40497 2024-12-06T01:45:36,807 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=40497 2024-12-06T01:45:36,899 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/00c764f7c3aa:0 server-side Connection retries=45 2024-12-06T01:45:36,901 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-06T01:45:36,901 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-12-06T01:45:36,902 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-12-06T01:45:36,902 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-06T01:45:36,902 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-12-06T01:45:36,905 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-12-06T01:45:36,908 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-12-06T01:45:36,909 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:37845 2024-12-06T01:45:36,911 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:37845 connecting to ZooKeeper ensemble=127.0.0.1:54114 2024-12-06T01:45:36,912 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-12-06T01:45:36,913 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-12-06T01:45:36,917 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:378450x0, quorum=127.0.0.1:54114, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-06T01:45:36,918 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:378450x0, quorum=127.0.0.1:54114, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-06T01:45:36,918 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:37845-0x100650089b40001 connected 2024-12-06T01:45:36,924 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-12-06T01:45:36,940 INFO [Time-limited test {}] bucket.FileIOEngine(99): Allocating cache 400 MB, on the path:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/eff72b7a-3322-3fdb-88e1-7287cd98d0d4/bucket.cache 2024-12-06T01:45:36,948 INFO [Time-limited test {}] bucket.BucketCache(364): Instantiating BucketCache with acceptableFactor: 0.95, minFactor: 0.85, extraFreeFactor: 0.1, singleFactor: 0.25, multiFactor: 0.5, memoryFactor: 0.25, useStrongRef: false 2024-12-06T01:45:36,950 INFO [Time-limited test {}] bucket.BucketCache(478): Starting BucketCachePersisterThread 2024-12-06T01:45:36,952 INFO [Time-limited test {}] bucket.BucketCachePersister(38): BucketCachePersister started with interval: 100 2024-12-06T01:45:36,955 INFO [Thread-10 {}] bucket.BucketCache(1530): Started retrieving bucket cache from file 2024-12-06T01:45:36,955 WARN [Thread-10 {}] bucket.BucketCache(1533): Persistence file missing! It's ok if it's first run after enabling persistent cache. 2024-12-06T01:45:36,962 INFO [Time-limited test {}] bucket.BucketCache(403): Started bucket cache; ioengine=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/eff72b7a-3322-3fdb-88e1-7287cd98d0d4/bucket.cache, capacity=400 MB, blockSize=64 KB, writerThreadNum=3, writerQLen=64, persistencePath=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/eff72b7a-3322-3fdb-88e1-7287cd98d0d4/bucket.persistence, bucketAllocator=org.apache.hadoop.hbase.io.hfile.bucket.BucketAllocator 2024-12-06T01:45:36,964 INFO [Thread-10 {}] bucket.BucketAllocator(343): Cache totalSize=418148352, buckets=199, bucket capacity=2101248=(4*525312)=(FEWEST_ITEMS_IN_BUCKET*(largest configured bucketcache size)) 2024-12-06T01:45:36,964 INFO [Thread-10 {}] bucket.BucketCache(413): Persistent bucket cache recovery from /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/eff72b7a-3322-3fdb-88e1-7287cd98d0d4/bucket.persistence is complete. 2024-12-06T01:45:36,970 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-12-06T01:45:36,973 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:37845-0x100650089b40001, quorum=127.0.0.1:54114, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-12-06T01:45:36,979 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:37845-0x100650089b40001, quorum=127.0.0.1:54114, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-12-06T01:45:36,980 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=37845 2024-12-06T01:45:36,980 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=37845 2024-12-06T01:45:36,981 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=37845 2024-12-06T01:45:36,982 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=37845 2024-12-06T01:45:36,982 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=37845 2024-12-06T01:45:37,003 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/00c764f7c3aa:0 server-side Connection retries=45 2024-12-06T01:45:37,003 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-06T01:45:37,004 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-12-06T01:45:37,004 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-12-06T01:45:37,004 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-06T01:45:37,005 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-12-06T01:45:37,005 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-12-06T01:45:37,005 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-12-06T01:45:37,008 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:44363 2024-12-06T01:45:37,011 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:44363 connecting to ZooKeeper ensemble=127.0.0.1:54114 2024-12-06T01:45:37,012 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-12-06T01:45:37,013 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-12-06T01:45:37,018 DEBUG [pool-21-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:443630x0, quorum=127.0.0.1:54114, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-06T01:45:37,019 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:44363-0x100650089b40002 connected 2024-12-06T01:45:37,019 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:44363-0x100650089b40002, quorum=127.0.0.1:54114, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-06T01:45:37,020 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-12-06T01:45:37,021 INFO [Time-limited test {}] bucket.FileIOEngine(99): Allocating cache 400 MB, on the path:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/eff72b7a-3322-3fdb-88e1-7287cd98d0d4/bucket.cache 2024-12-06T01:45:37,022 INFO [Time-limited test {}] bucket.BucketCache(364): Instantiating BucketCache with acceptableFactor: 0.95, minFactor: 0.85, extraFreeFactor: 0.1, singleFactor: 0.25, multiFactor: 0.5, memoryFactor: 0.25, useStrongRef: false 2024-12-06T01:45:37,022 INFO [Time-limited test {}] bucket.BucketCache(478): Starting BucketCachePersisterThread 2024-12-06T01:45:37,022 INFO [Time-limited test {}] bucket.BucketCachePersister(38): BucketCachePersister started with interval: 100 2024-12-06T01:45:37,024 INFO [Thread-11 {}] bucket.BucketCache(1530): Started retrieving bucket cache from file 2024-12-06T01:45:37,024 WARN [Thread-11 {}] bucket.BucketCache(1533): Persistence file missing! It's ok if it's first run after enabling persistent cache. 2024-12-06T01:45:37,024 INFO [Time-limited test {}] bucket.BucketCache(403): Started bucket cache; ioengine=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/eff72b7a-3322-3fdb-88e1-7287cd98d0d4/bucket.cache, capacity=400 MB, blockSize=64 KB, writerThreadNum=3, writerQLen=64, persistencePath=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/eff72b7a-3322-3fdb-88e1-7287cd98d0d4/bucket.persistence, bucketAllocator=org.apache.hadoop.hbase.io.hfile.bucket.BucketAllocator 2024-12-06T01:45:37,025 INFO [Thread-11 {}] bucket.BucketAllocator(343): Cache totalSize=418148352, buckets=199, bucket capacity=2101248=(4*525312)=(FEWEST_ITEMS_IN_BUCKET*(largest configured bucketcache size)) 2024-12-06T01:45:37,025 INFO [Thread-11 {}] bucket.BucketCache(413): Persistent bucket cache recovery from /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/eff72b7a-3322-3fdb-88e1-7287cd98d0d4/bucket.persistence is complete. 2024-12-06T01:45:37,028 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-12-06T01:45:37,030 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:44363-0x100650089b40002, quorum=127.0.0.1:54114, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-12-06T01:45:37,033 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:44363-0x100650089b40002, quorum=127.0.0.1:54114, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-12-06T01:45:37,034 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=44363 2024-12-06T01:45:37,036 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=44363 2024-12-06T01:45:37,039 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=44363 2024-12-06T01:45:37,041 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=44363 2024-12-06T01:45:37,041 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=44363 2024-12-06T01:45:37,066 DEBUG [M:0;00c764f7c3aa:40497 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;00c764f7c3aa:40497 2024-12-06T01:45:37,068 INFO [master/00c764f7c3aa:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/00c764f7c3aa,40497,1733449536009 2024-12-06T01:45:37,078 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37845-0x100650089b40001, quorum=127.0.0.1:54114, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-06T01:45:37,078 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40497-0x100650089b40000, quorum=127.0.0.1:54114, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-06T01:45:37,078 DEBUG [pool-21-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44363-0x100650089b40002, quorum=127.0.0.1:54114, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-06T01:45:37,082 DEBUG [master/00c764f7c3aa:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:40497-0x100650089b40000, quorum=127.0.0.1:54114, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/00c764f7c3aa,40497,1733449536009 2024-12-06T01:45:37,116 DEBUG [pool-21-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44363-0x100650089b40002, quorum=127.0.0.1:54114, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-12-06T01:45:37,116 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37845-0x100650089b40001, quorum=127.0.0.1:54114, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-12-06T01:45:37,116 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40497-0x100650089b40000, quorum=127.0.0.1:54114, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-06T01:45:37,117 DEBUG [pool-21-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44363-0x100650089b40002, quorum=127.0.0.1:54114, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-06T01:45:37,117 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37845-0x100650089b40001, quorum=127.0.0.1:54114, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-06T01:45:37,119 DEBUG [master/00c764f7c3aa:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:40497-0x100650089b40000, quorum=127.0.0.1:54114, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-12-06T01:45:37,121 INFO [master/00c764f7c3aa:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/00c764f7c3aa,40497,1733449536009 from backup master directory 2024-12-06T01:45:37,124 DEBUG [pool-21-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44363-0x100650089b40002, quorum=127.0.0.1:54114, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-06T01:45:37,124 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37845-0x100650089b40001, quorum=127.0.0.1:54114, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-06T01:45:37,124 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40497-0x100650089b40000, quorum=127.0.0.1:54114, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/00c764f7c3aa,40497,1733449536009 2024-12-06T01:45:37,125 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40497-0x100650089b40000, quorum=127.0.0.1:54114, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-06T01:45:37,126 WARN [master/00c764f7c3aa:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-12-06T01:45:37,127 INFO [master/00c764f7c3aa:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=00c764f7c3aa,40497,1733449536009 2024-12-06T01:45:37,129 INFO [master/00c764f7c3aa:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating data MemStoreChunkPool with chunk size 2 MB, max count 396, initial count 0 2024-12-06T01:45:37,131 INFO [master/00c764f7c3aa:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating index MemStoreChunkPool with chunk size 204.80 KB, max count 440, initial count 0 2024-12-06T01:45:37,138 DEBUG [master/00c764f7c3aa:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/eff72b7a-3322-3fdb-88e1-7287cd98d0d4/e93bd390-0caa-6c00-f03a-52237a196617/hbase.id] with ID: 0649e6ab-29a8-4257-8352-d4d974c2a6de 2024-12-06T01:45:37,139 DEBUG [master/00c764f7c3aa:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/eff72b7a-3322-3fdb-88e1-7287cd98d0d4/e93bd390-0caa-6c00-f03a-52237a196617/.tmp/hbase.id 2024-12-06T01:45:37,145 DEBUG [master/00c764f7c3aa:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/eff72b7a-3322-3fdb-88e1-7287cd98d0d4/e93bd390-0caa-6c00-f03a-52237a196617/.tmp/hbase.id]:[file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/eff72b7a-3322-3fdb-88e1-7287cd98d0d4/e93bd390-0caa-6c00-f03a-52237a196617/hbase.id] 2024-12-06T01:45:37,190 DEBUG [master/00c764f7c3aa:0:becomeActiveMaster {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-12-06T01:45:37,196 INFO [master/00c764f7c3aa:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem. 2024-12-06T01:45:37,225 INFO [master/00c764f7c3aa:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 26ms. 2024-12-06T01:45:37,229 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37845-0x100650089b40001, quorum=127.0.0.1:54114, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-06T01:45:37,229 DEBUG [pool-21-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44363-0x100650089b40002, quorum=127.0.0.1:54114, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-06T01:45:37,230 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40497-0x100650089b40000, quorum=127.0.0.1:54114, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-06T01:45:37,267 INFO [master/00c764f7c3aa:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-06T01:45:37,272 INFO [master/00c764f7c3aa:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-12-06T01:45:37,327 DEBUG [master/00c764f7c3aa:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(244): No decryptEncryptedDataEncryptionKey method in DFSClient, should be hadoop version with HDFS-12396 java.lang.NoSuchMethodException: org.apache.hadoop.hdfs.DFSClient.decryptEncryptedDataEncryptionKey(org.apache.hadoop.fs.FileEncryptionInfo) at java.lang.Class.getDeclaredMethod(Class.java:2675) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelperWithoutHDFS12396(FanOutOneBlockAsyncDFSOutputSaslHelper.java:183) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelper(FanOutOneBlockAsyncDFSOutputSaslHelper.java:242) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.(FanOutOneBlockAsyncDFSOutputSaslHelper.java:253) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.lang.Class.forName0(Native Method) ~[?:?] at java.lang.Class.forName(Class.java:375) ~[?:?] at org.apache.hadoop.hbase.wal.AsyncFSWALProvider.load(AsyncFSWALProvider.java:150) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.getProviderClass(WALFactory.java:174) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:262) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:231) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegion.create(MasterRegion.java:400) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegionFactory.create(MasterRegionFactory.java:135) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.finishActiveMasterInitialization(HMaster.java:1003) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.startActiveMasterManager(HMaster.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.lambda$run$0(HMaster.java:613) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.lambda$tracedRunnable$2(TraceUtil.java:155) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T01:45:37,333 INFO [master/00c764f7c3aa:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-06T01:45:37,402 INFO [master/00c764f7c3aa:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/eff72b7a-3322-3fdb-88e1-7287cd98d0d4/e93bd390-0caa-6c00-f03a-52237a196617/MasterData/data/master/store 2024-12-06T01:45:37,428 INFO [master/00c764f7c3aa:0:becomeActiveMaster {}] throttle.StoreHotnessProtector(112): StoreHotnessProtector is disabled. Set hbase.region.store.parallel.put.limit > 0 to enable, which may help mitigate load under heavy write pressure. 2024-12-06T01:45:37,433 DEBUG [master/00c764f7c3aa:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-06T01:45:37,434 DEBUG [master/00c764f7c3aa:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-12-06T01:45:37,435 INFO [master/00c764f7c3aa:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-06T01:45:37,435 DEBUG [master/00c764f7c3aa:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-06T01:45:37,437 DEBUG [master/00c764f7c3aa:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-12-06T01:45:37,438 DEBUG [master/00c764f7c3aa:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-06T01:45:37,438 INFO [master/00c764f7c3aa:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-06T01:45:37,439 DEBUG [master/00c764f7c3aa:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1733449537434Disabling compacts and flushes for region at 1733449537434Disabling writes for close at 1733449537437 (+3 ms)Writing region close event to WAL at 1733449537438 (+1 ms)Closed at 1733449537438 2024-12-06T01:45:37,440 WARN [master/00c764f7c3aa:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/eff72b7a-3322-3fdb-88e1-7287cd98d0d4/e93bd390-0caa-6c00-f03a-52237a196617/MasterData/data/master/store/.initializing 2024-12-06T01:45:37,440 DEBUG [master/00c764f7c3aa:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/eff72b7a-3322-3fdb-88e1-7287cd98d0d4/e93bd390-0caa-6c00-f03a-52237a196617/MasterData/WALs/00c764f7c3aa,40497,1733449536009 2024-12-06T01:45:37,457 INFO [master/00c764f7c3aa:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-06T01:45:37,475 INFO [master/00c764f7c3aa:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=00c764f7c3aa%2C40497%2C1733449536009, suffix=, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/eff72b7a-3322-3fdb-88e1-7287cd98d0d4/e93bd390-0caa-6c00-f03a-52237a196617/MasterData/WALs/00c764f7c3aa,40497,1733449536009, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/eff72b7a-3322-3fdb-88e1-7287cd98d0d4/e93bd390-0caa-6c00-f03a-52237a196617/MasterData/oldWALs, maxLogs=10 2024-12-06T01:45:37,526 INFO [master/00c764f7c3aa:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/eff72b7a-3322-3fdb-88e1-7287cd98d0d4/e93bd390-0caa-6c00-f03a-52237a196617/MasterData/WALs/00c764f7c3aa,40497,1733449536009/00c764f7c3aa%2C40497%2C1733449536009.1733449537481 2024-12-06T01:45:37,527 DEBUG [master/00c764f7c3aa:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-12-06T01:45:37,528 DEBUG [master/00c764f7c3aa:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-12-06T01:45:37,529 DEBUG [master/00c764f7c3aa:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-06T01:45:37,533 DEBUG [master/00c764f7c3aa:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-12-06T01:45:37,535 DEBUG [master/00c764f7c3aa:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-12-06T01:45:37,582 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-12-06T01:45:37,617 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-12-06T01:45:37,621 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T01:45:37,623 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-06T01:45:37,624 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-12-06T01:45:37,630 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-12-06T01:45:37,630 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T01:45:37,630 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-06T01:45:37,631 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-12-06T01:45:37,636 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-12-06T01:45:37,636 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T01:45:37,637 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-06T01:45:37,637 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-12-06T01:45:37,643 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-12-06T01:45:37,643 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T01:45:37,643 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-06T01:45:37,644 DEBUG [master/00c764f7c3aa:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682 2024-12-06T01:45:37,647 DEBUG [master/00c764f7c3aa:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/eff72b7a-3322-3fdb-88e1-7287cd98d0d4/e93bd390-0caa-6c00-f03a-52237a196617/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-12-06T01:45:37,648 DEBUG [master/00c764f7c3aa:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/eff72b7a-3322-3fdb-88e1-7287cd98d0d4/e93bd390-0caa-6c00-f03a-52237a196617/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-12-06T01:45:37,651 DEBUG [master/00c764f7c3aa:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682 2024-12-06T01:45:37,652 DEBUG [master/00c764f7c3aa:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682 2024-12-06T01:45:37,654 DEBUG [master/00c764f7c3aa:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-12-06T01:45:37,656 DEBUG [master/00c764f7c3aa:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-12-06T01:45:37,665 DEBUG [master/00c764f7c3aa:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/eff72b7a-3322-3fdb-88e1-7287cd98d0d4/e93bd390-0caa-6c00-f03a-52237a196617/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-06T01:45:37,666 INFO [master/00c764f7c3aa:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=59871720, jitterRate=-0.1078418493270874}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-12-06T01:45:37,674 DEBUG [master/00c764f7c3aa:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682: Writing region info on filesystem at 1733449537549Initializing all the Stores at 1733449537550 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733449537550Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733449537551 (+1 ms)Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733449537552 (+1 ms)Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733449537552Cleaning up temporary data from old regions at 1733449537652 (+100 ms)Region opened successfully at 1733449537674 (+22 ms) 2024-12-06T01:45:37,676 INFO [master/00c764f7c3aa:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-12-06T01:45:37,722 DEBUG [master/00c764f7c3aa:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@d7128c4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=00c764f7c3aa/172.17.0.2:0 2024-12-06T01:45:37,767 INFO [master/00c764f7c3aa:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating... 2024-12-06T01:45:37,780 INFO [master/00c764f7c3aa:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5 2024-12-06T01:45:37,781 INFO [master/00c764f7c3aa:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50 2024-12-06T01:45:37,784 INFO [master/00c764f7c3aa:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-12-06T01:45:37,785 INFO [master/00c764f7c3aa:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-12-06T01:45:37,791 INFO [master/00c764f7c3aa:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 6 msec 2024-12-06T01:45:37,792 INFO [master/00c764f7c3aa:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-12-06T01:45:37,832 INFO [master/00c764f7c3aa:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2024-12-06T01:45:37,841 DEBUG [master/00c764f7c3aa:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:40497-0x100650089b40000, quorum=127.0.0.1:54114, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error) 2024-12-06T01:45:37,844 DEBUG [master/00c764f7c3aa:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false 2024-12-06T01:45:37,847 INFO [master/00c764f7c3aa:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2024-12-06T01:45:37,848 DEBUG [master/00c764f7c3aa:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:40497-0x100650089b40000, quorum=127.0.0.1:54114, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error) 2024-12-06T01:45:37,850 DEBUG [master/00c764f7c3aa:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false 2024-12-06T01:45:37,852 INFO [master/00c764f7c3aa:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2024-12-06T01:45:37,856 DEBUG [master/00c764f7c3aa:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:40497-0x100650089b40000, quorum=127.0.0.1:54114, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error) 2024-12-06T01:45:37,858 DEBUG [master/00c764f7c3aa:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false 2024-12-06T01:45:37,860 DEBUG [master/00c764f7c3aa:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:40497-0x100650089b40000, quorum=127.0.0.1:54114, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error) 2024-12-06T01:45:37,862 DEBUG [master/00c764f7c3aa:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false 2024-12-06T01:45:37,883 DEBUG [master/00c764f7c3aa:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:40497-0x100650089b40000, quorum=127.0.0.1:54114, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error) 2024-12-06T01:45:37,885 DEBUG [master/00c764f7c3aa:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false 2024-12-06T01:45:37,903 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40497-0x100650089b40000, quorum=127.0.0.1:54114, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-12-06T01:45:37,903 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37845-0x100650089b40001, quorum=127.0.0.1:54114, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-12-06T01:45:37,903 DEBUG [pool-21-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44363-0x100650089b40002, quorum=127.0.0.1:54114, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-12-06T01:45:37,903 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40497-0x100650089b40000, quorum=127.0.0.1:54114, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-06T01:45:37,903 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37845-0x100650089b40001, quorum=127.0.0.1:54114, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-06T01:45:37,903 DEBUG [pool-21-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44363-0x100650089b40002, quorum=127.0.0.1:54114, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-06T01:45:37,907 INFO [master/00c764f7c3aa:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=00c764f7c3aa,40497,1733449536009, sessionid=0x100650089b40000, setting cluster-up flag (Was=false) 2024-12-06T01:45:37,922 DEBUG [pool-21-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44363-0x100650089b40002, quorum=127.0.0.1:54114, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-06T01:45:37,922 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40497-0x100650089b40000, quorum=127.0.0.1:54114, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-06T01:45:37,922 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37845-0x100650089b40001, quorum=127.0.0.1:54114, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-06T01:45:37,929 DEBUG [master/00c764f7c3aa:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort 2024-12-06T01:45:37,931 DEBUG [master/00c764f7c3aa:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=00c764f7c3aa,40497,1733449536009 2024-12-06T01:45:37,936 DEBUG [pool-21-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44363-0x100650089b40002, quorum=127.0.0.1:54114, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-06T01:45:37,936 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40497-0x100650089b40000, quorum=127.0.0.1:54114, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-06T01:45:37,936 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37845-0x100650089b40001, quorum=127.0.0.1:54114, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-06T01:45:37,943 DEBUG [master/00c764f7c3aa:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort 2024-12-06T01:45:37,944 DEBUG [master/00c764f7c3aa:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=00c764f7c3aa,40497,1733449536009 2024-12-06T01:45:37,950 INFO [master/00c764f7c3aa:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/eff72b7a-3322-3fdb-88e1-7287cd98d0d4/e93bd390-0caa-6c00-f03a-52237a196617/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again 2024-12-06T01:45:38,036 DEBUG [master/00c764f7c3aa:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta 2024-12-06T01:45:38,046 INFO [RS:0;00c764f7c3aa:37845 {}] regionserver.HRegionServer(746): ClusterId : 0649e6ab-29a8-4257-8352-d4d974c2a6de 2024-12-06T01:45:38,046 INFO [RS:1;00c764f7c3aa:44363 {}] regionserver.HRegionServer(746): ClusterId : 0649e6ab-29a8-4257-8352-d4d974c2a6de 2024-12-06T01:45:38,048 INFO [master/00c764f7c3aa:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2 2024-12-06T01:45:38,050 DEBUG [RS:1;00c764f7c3aa:44363 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-12-06T01:45:38,050 DEBUG [RS:0;00c764f7c3aa:37845 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-12-06T01:45:38,057 DEBUG [RS:1;00c764f7c3aa:44363 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-12-06T01:45:38,057 DEBUG [RS:0;00c764f7c3aa:37845 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-12-06T01:45:38,057 DEBUG [RS:1;00c764f7c3aa:44363 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-12-06T01:45:38,057 DEBUG [RS:0;00c764f7c3aa:37845 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-12-06T01:45:38,059 INFO [master/00c764f7c3aa:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2024-12-06T01:45:38,061 DEBUG [RS:1;00c764f7c3aa:44363 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-12-06T01:45:38,061 DEBUG [RS:0;00c764f7c3aa:37845 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-12-06T01:45:38,062 DEBUG [RS:1;00c764f7c3aa:44363 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5d18eedb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=00c764f7c3aa/172.17.0.2:0 2024-12-06T01:45:38,062 DEBUG [RS:0;00c764f7c3aa:37845 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@15b495ff, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=00c764f7c3aa/172.17.0.2:0 2024-12-06T01:45:38,067 DEBUG [master/00c764f7c3aa:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: 00c764f7c3aa,40497,1733449536009 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2024-12-06T01:45:38,076 DEBUG [master/00c764f7c3aa:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/00c764f7c3aa:0, corePoolSize=5, maxPoolSize=5 2024-12-06T01:45:38,076 DEBUG [master/00c764f7c3aa:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/00c764f7c3aa:0, corePoolSize=5, maxPoolSize=5 2024-12-06T01:45:38,076 DEBUG [master/00c764f7c3aa:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/00c764f7c3aa:0, corePoolSize=5, maxPoolSize=5 2024-12-06T01:45:38,077 DEBUG [master/00c764f7c3aa:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/00c764f7c3aa:0, corePoolSize=5, maxPoolSize=5 2024-12-06T01:45:38,077 DEBUG [master/00c764f7c3aa:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/00c764f7c3aa:0, corePoolSize=10, maxPoolSize=10 2024-12-06T01:45:38,077 DEBUG [master/00c764f7c3aa:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/00c764f7c3aa:0, corePoolSize=1, maxPoolSize=1 2024-12-06T01:45:38,077 DEBUG [master/00c764f7c3aa:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/00c764f7c3aa:0, corePoolSize=2, maxPoolSize=2 2024-12-06T01:45:38,077 DEBUG [master/00c764f7c3aa:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/00c764f7c3aa:0, corePoolSize=1, maxPoolSize=1 2024-12-06T01:45:38,079 DEBUG [RS:0;00c764f7c3aa:37845 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;00c764f7c3aa:37845 2024-12-06T01:45:38,082 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta 2024-12-06T01:45:38,083 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region 2024-12-06T01:45:38,084 INFO [RS:0;00c764f7c3aa:37845 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-12-06T01:45:38,085 INFO [RS:0;00c764f7c3aa:37845 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-12-06T01:45:38,086 DEBUG [RS:0;00c764f7c3aa:37845 {}] regionserver.HRegionServer(832): About to register with Master. 2024-12-06T01:45:38,087 DEBUG [RS:1;00c764f7c3aa:44363 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:1;00c764f7c3aa:44363 2024-12-06T01:45:38,087 INFO [RS:1;00c764f7c3aa:44363 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-12-06T01:45:38,088 INFO [RS:1;00c764f7c3aa:44363 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-12-06T01:45:38,088 DEBUG [RS:1;00c764f7c3aa:44363 {}] regionserver.HRegionServer(832): About to register with Master. 2024-12-06T01:45:38,089 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T01:45:38,090 INFO [RS:0;00c764f7c3aa:37845 {}] regionserver.HRegionServer(2659): reportForDuty to master=00c764f7c3aa,40497,1733449536009 with port=37845, startcode=1733449536857 2024-12-06T01:45:38,089 INFO [RS:1;00c764f7c3aa:44363 {}] regionserver.HRegionServer(2659): reportForDuty to master=00c764f7c3aa,40497,1733449536009 with port=44363, startcode=1733449537002 2024-12-06T01:45:38,089 INFO [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-12-06T01:45:38,091 INFO [master/00c764f7c3aa:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1733449568091 2024-12-06T01:45:38,093 INFO [master/00c764f7c3aa:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2024-12-06T01:45:38,095 INFO [master/00c764f7c3aa:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2024-12-06T01:45:38,099 INFO [master/00c764f7c3aa:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2024-12-06T01:45:38,099 INFO [master/00c764f7c3aa:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2024-12-06T01:45:38,100 INFO [master/00c764f7c3aa:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2024-12-06T01:45:38,100 INFO [master/00c764f7c3aa:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2024-12-06T01:45:38,101 INFO [master/00c764f7c3aa:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-12-06T01:45:38,105 INFO [master/00c764f7c3aa:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2024-12-06T01:45:38,106 INFO [master/00c764f7c3aa:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2024-12-06T01:45:38,107 INFO [master/00c764f7c3aa:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2024-12-06T01:45:38,107 DEBUG [RS:0;00c764f7c3aa:37845 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-12-06T01:45:38,107 DEBUG [RS:1;00c764f7c3aa:44363 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-12-06T01:45:38,111 INFO [master/00c764f7c3aa:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2024-12-06T01:45:38,111 INFO [master/00c764f7c3aa:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2024-12-06T01:45:38,113 DEBUG [master/00c764f7c3aa:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/00c764f7c3aa:0:becomeActiveMaster-HFileCleaner.large.0-1733449538112,5,FailOnTimeoutGroup] 2024-12-06T01:45:38,114 DEBUG [master/00c764f7c3aa:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/00c764f7c3aa:0:becomeActiveMaster-HFileCleaner.small.0-1733449538113,5,FailOnTimeoutGroup] 2024-12-06T01:45:38,114 INFO [master/00c764f7c3aa:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-12-06T01:45:38,114 INFO [master/00c764f7c3aa:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2024-12-06T01:45:38,115 INFO [master/00c764f7c3aa:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2024-12-06T01:45:38,116 INFO [master/00c764f7c3aa:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2024-12-06T01:45:38,120 INFO [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/eff72b7a-3322-3fdb-88e1-7287cd98d0d4/e93bd390-0caa-6c00-f03a-52237a196617/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321 2024-12-06T01:45:38,121 INFO [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/eff72b7a-3322-3fdb-88e1-7287cd98d0d4/e93bd390-0caa-6c00-f03a-52237a196617 2024-12-06T01:45:38,130 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-06T01:45:38,132 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family info of region 1588230740 2024-12-06T01:45:38,136 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-12-06T01:45:38,137 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T01:45:38,137 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-06T01:45:38,137 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family ns of region 1588230740 2024-12-06T01:45:38,141 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-12-06T01:45:38,141 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T01:45:38,142 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-06T01:45:38,142 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family rep_barrier of region 1588230740 2024-12-06T01:45:38,150 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-12-06T01:45:38,151 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T01:45:38,151 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-06T01:45:38,151 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family table of region 1588230740 2024-12-06T01:45:38,157 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-12-06T01:45:38,158 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T01:45:38,158 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-06T01:45:38,158 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-12-06T01:45:38,160 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/eff72b7a-3322-3fdb-88e1-7287cd98d0d4/e93bd390-0caa-6c00-f03a-52237a196617/data/hbase/meta/1588230740 2024-12-06T01:45:38,160 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/eff72b7a-3322-3fdb-88e1-7287cd98d0d4/e93bd390-0caa-6c00-f03a-52237a196617/data/hbase/meta/1588230740 2024-12-06T01:45:38,161 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-12-06T01:45:38,161 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-12-06T01:45:38,161 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-12-06T01:45:38,163 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-12-06T01:45:38,175 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/eff72b7a-3322-3fdb-88e1-7287cd98d0d4/e93bd390-0caa-6c00-f03a-52237a196617/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-06T01:45:38,175 INFO [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=68424905, jitterRate=0.019610539078712463}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-12-06T01:45:38,179 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740: Writing region info on filesystem at 1733449538130Initializing all the Stores at 1733449538130Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733449538130Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733449538132 (+2 ms)Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733449538132Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733449538132Cleaning up temporary data from old regions at 1733449538161 (+29 ms)Region opened successfully at 1733449538179 (+18 ms) 2024-12-06T01:45:38,179 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-12-06T01:45:38,180 INFO [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-12-06T01:45:38,180 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-12-06T01:45:38,180 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-12-06T01:45:38,180 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-12-06T01:45:38,182 INFO [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-12-06T01:45:38,182 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1733449538179Disabling compacts and flushes for region at 1733449538179Disabling writes for close at 1733449538180 (+1 ms)Writing region close event to WAL at 1733449538182 (+2 ms)Closed at 1733449538182 2024-12-06T01:45:38,185 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta 2024-12-06T01:45:38,186 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta 2024-12-06T01:45:38,190 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57539, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=RegionServerStatusService 2024-12-06T01:45:38,190 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39993, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=RegionServerStatusService 2024-12-06T01:45:38,197 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-12-06T01:45:38,198 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=40497 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 00c764f7c3aa,37845,1733449536857 2024-12-06T01:45:38,201 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=40497 {}] master.ServerManager(517): Registering regionserver=00c764f7c3aa,37845,1733449536857 2024-12-06T01:45:38,207 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-12-06T01:45:38,210 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2024-12-06T01:45:38,215 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=40497 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 00c764f7c3aa,44363,1733449537002 2024-12-06T01:45:38,215 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=40497 {}] master.ServerManager(517): Registering regionserver=00c764f7c3aa,44363,1733449537002 2024-12-06T01:45:38,219 DEBUG [RS:1;00c764f7c3aa:44363 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/eff72b7a-3322-3fdb-88e1-7287cd98d0d4/e93bd390-0caa-6c00-f03a-52237a196617 2024-12-06T01:45:38,219 DEBUG [RS:1;00c764f7c3aa:44363 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=file:/ 2024-12-06T01:45:38,220 DEBUG [RS:1;00c764f7c3aa:44363 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-12-06T01:45:38,220 DEBUG [RS:0;00c764f7c3aa:37845 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/eff72b7a-3322-3fdb-88e1-7287cd98d0d4/e93bd390-0caa-6c00-f03a-52237a196617 2024-12-06T01:45:38,220 DEBUG [RS:0;00c764f7c3aa:37845 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=file:/ 2024-12-06T01:45:38,220 DEBUG [RS:0;00c764f7c3aa:37845 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-12-06T01:45:38,227 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40497-0x100650089b40000, quorum=127.0.0.1:54114, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-12-06T01:45:38,228 DEBUG [RS:0;00c764f7c3aa:37845 {}] zookeeper.ZKUtil(111): regionserver:37845-0x100650089b40001, quorum=127.0.0.1:54114, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/00c764f7c3aa,37845,1733449536857 2024-12-06T01:45:38,228 WARN [RS:0;00c764f7c3aa:37845 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-12-06T01:45:38,228 DEBUG [RS:1;00c764f7c3aa:44363 {}] zookeeper.ZKUtil(111): regionserver:44363-0x100650089b40002, quorum=127.0.0.1:54114, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/00c764f7c3aa,44363,1733449537002 2024-12-06T01:45:38,228 INFO [RS:0;00c764f7c3aa:37845 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-06T01:45:38,228 WARN [RS:1;00c764f7c3aa:44363 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-12-06T01:45:38,229 INFO [RS:1;00c764f7c3aa:44363 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-06T01:45:38,229 DEBUG [RS:0;00c764f7c3aa:37845 {}] regionserver.HRegionServer(1793): logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/eff72b7a-3322-3fdb-88e1-7287cd98d0d4/e93bd390-0caa-6c00-f03a-52237a196617/WALs/00c764f7c3aa,37845,1733449536857 2024-12-06T01:45:38,229 DEBUG [RS:1;00c764f7c3aa:44363 {}] regionserver.HRegionServer(1793): logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/eff72b7a-3322-3fdb-88e1-7287cd98d0d4/e93bd390-0caa-6c00-f03a-52237a196617/WALs/00c764f7c3aa,44363,1733449537002 2024-12-06T01:45:38,231 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [00c764f7c3aa,37845,1733449536857] 2024-12-06T01:45:38,231 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [00c764f7c3aa,44363,1733449537002] 2024-12-06T01:45:38,263 INFO [RS:1;00c764f7c3aa:44363 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-12-06T01:45:38,263 INFO [RS:0;00c764f7c3aa:37845 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-12-06T01:45:38,283 INFO [RS:1;00c764f7c3aa:44363 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-12-06T01:45:38,283 INFO [RS:0;00c764f7c3aa:37845 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-12-06T01:45:38,289 INFO [RS:0;00c764f7c3aa:37845 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-12-06T01:45:38,289 INFO [RS:1;00c764f7c3aa:44363 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-12-06T01:45:38,289 INFO [RS:0;00c764f7c3aa:37845 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-12-06T01:45:38,289 INFO [RS:1;00c764f7c3aa:44363 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-12-06T01:45:38,293 INFO [RS:1;00c764f7c3aa:44363 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-12-06T01:45:38,293 INFO [RS:0;00c764f7c3aa:37845 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-12-06T01:45:38,299 INFO [RS:1;00c764f7c3aa:44363 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-12-06T01:45:38,299 INFO [RS:0;00c764f7c3aa:37845 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-12-06T01:45:38,301 INFO [RS:0;00c764f7c3aa:37845 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-12-06T01:45:38,301 INFO [RS:1;00c764f7c3aa:44363 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-12-06T01:45:38,302 DEBUG [RS:1;00c764f7c3aa:44363 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/00c764f7c3aa:0, corePoolSize=1, maxPoolSize=1 2024-12-06T01:45:38,302 DEBUG [RS:0;00c764f7c3aa:37845 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/00c764f7c3aa:0, corePoolSize=1, maxPoolSize=1 2024-12-06T01:45:38,302 DEBUG [RS:1;00c764f7c3aa:44363 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/00c764f7c3aa:0, corePoolSize=1, maxPoolSize=1 2024-12-06T01:45:38,302 DEBUG [RS:0;00c764f7c3aa:37845 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/00c764f7c3aa:0, corePoolSize=1, maxPoolSize=1 2024-12-06T01:45:38,302 DEBUG [RS:0;00c764f7c3aa:37845 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/00c764f7c3aa:0, corePoolSize=1, maxPoolSize=1 2024-12-06T01:45:38,302 DEBUG [RS:1;00c764f7c3aa:44363 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/00c764f7c3aa:0, corePoolSize=1, maxPoolSize=1 2024-12-06T01:45:38,302 DEBUG [RS:0;00c764f7c3aa:37845 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/00c764f7c3aa:0, corePoolSize=1, maxPoolSize=1 2024-12-06T01:45:38,302 DEBUG [RS:1;00c764f7c3aa:44363 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/00c764f7c3aa:0, corePoolSize=1, maxPoolSize=1 2024-12-06T01:45:38,303 DEBUG [RS:0;00c764f7c3aa:37845 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/00c764f7c3aa:0, corePoolSize=1, maxPoolSize=1 2024-12-06T01:45:38,303 DEBUG [RS:1;00c764f7c3aa:44363 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/00c764f7c3aa:0, corePoolSize=1, maxPoolSize=1 2024-12-06T01:45:38,303 DEBUG [RS:1;00c764f7c3aa:44363 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/00c764f7c3aa:0, corePoolSize=2, maxPoolSize=2 2024-12-06T01:45:38,303 DEBUG [RS:0;00c764f7c3aa:37845 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/00c764f7c3aa:0, corePoolSize=2, maxPoolSize=2 2024-12-06T01:45:38,303 DEBUG [RS:0;00c764f7c3aa:37845 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/00c764f7c3aa:0, corePoolSize=1, maxPoolSize=1 2024-12-06T01:45:38,303 DEBUG [RS:1;00c764f7c3aa:44363 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/00c764f7c3aa:0, corePoolSize=1, maxPoolSize=1 2024-12-06T01:45:38,303 DEBUG [RS:0;00c764f7c3aa:37845 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/00c764f7c3aa:0, corePoolSize=1, maxPoolSize=1 2024-12-06T01:45:38,303 DEBUG [RS:1;00c764f7c3aa:44363 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/00c764f7c3aa:0, corePoolSize=1, maxPoolSize=1 2024-12-06T01:45:38,303 DEBUG [RS:0;00c764f7c3aa:37845 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/00c764f7c3aa:0, corePoolSize=1, maxPoolSize=1 2024-12-06T01:45:38,303 DEBUG [RS:1;00c764f7c3aa:44363 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/00c764f7c3aa:0, corePoolSize=1, maxPoolSize=1 2024-12-06T01:45:38,304 DEBUG [RS:0;00c764f7c3aa:37845 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/00c764f7c3aa:0, corePoolSize=1, maxPoolSize=1 2024-12-06T01:45:38,304 DEBUG [RS:1;00c764f7c3aa:44363 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/00c764f7c3aa:0, corePoolSize=1, maxPoolSize=1 2024-12-06T01:45:38,304 DEBUG [RS:1;00c764f7c3aa:44363 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/00c764f7c3aa:0, corePoolSize=1, maxPoolSize=1 2024-12-06T01:45:38,304 DEBUG [RS:0;00c764f7c3aa:37845 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/00c764f7c3aa:0, corePoolSize=1, maxPoolSize=1 2024-12-06T01:45:38,304 DEBUG [RS:1;00c764f7c3aa:44363 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/00c764f7c3aa:0, corePoolSize=1, maxPoolSize=1 2024-12-06T01:45:38,304 DEBUG [RS:0;00c764f7c3aa:37845 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/00c764f7c3aa:0, corePoolSize=1, maxPoolSize=1 2024-12-06T01:45:38,305 DEBUG [RS:0;00c764f7c3aa:37845 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/00c764f7c3aa:0, corePoolSize=3, maxPoolSize=3 2024-12-06T01:45:38,305 DEBUG [RS:1;00c764f7c3aa:44363 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/00c764f7c3aa:0, corePoolSize=3, maxPoolSize=3 2024-12-06T01:45:38,305 DEBUG [RS:0;00c764f7c3aa:37845 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/00c764f7c3aa:0, corePoolSize=3, maxPoolSize=3 2024-12-06T01:45:38,305 DEBUG [RS:1;00c764f7c3aa:44363 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/00c764f7c3aa:0, corePoolSize=3, maxPoolSize=3 2024-12-06T01:45:38,307 INFO [RS:0;00c764f7c3aa:37845 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-12-06T01:45:38,307 INFO [RS:0;00c764f7c3aa:37845 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-12-06T01:45:38,308 INFO [RS:0;00c764f7c3aa:37845 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-06T01:45:38,308 INFO [RS:0;00c764f7c3aa:37845 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-12-06T01:45:38,308 INFO [RS:1;00c764f7c3aa:44363 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-12-06T01:45:38,308 INFO [RS:0;00c764f7c3aa:37845 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-12-06T01:45:38,308 INFO [RS:1;00c764f7c3aa:44363 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-12-06T01:45:38,308 INFO [RS:0;00c764f7c3aa:37845 {}] hbase.ChoreService(168): Chore ScheduledChore name=00c764f7c3aa,37845,1733449536857-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-12-06T01:45:38,308 INFO [RS:1;00c764f7c3aa:44363 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-06T01:45:38,308 INFO [RS:1;00c764f7c3aa:44363 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-12-06T01:45:38,309 INFO [RS:1;00c764f7c3aa:44363 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-12-06T01:45:38,309 INFO [RS:1;00c764f7c3aa:44363 {}] hbase.ChoreService(168): Chore ScheduledChore name=00c764f7c3aa,44363,1733449537002-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-12-06T01:45:38,330 INFO [RS:0;00c764f7c3aa:37845 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-12-06T01:45:38,332 INFO [RS:0;00c764f7c3aa:37845 {}] hbase.ChoreService(168): Chore ScheduledChore name=00c764f7c3aa,37845,1733449536857-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-06T01:45:38,333 INFO [RS:0;00c764f7c3aa:37845 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-12-06T01:45:38,333 INFO [RS:0;00c764f7c3aa:37845 {}] regionserver.Replication(171): 00c764f7c3aa,37845,1733449536857 started 2024-12-06T01:45:38,335 INFO [RS:1;00c764f7c3aa:44363 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-12-06T01:45:38,336 INFO [RS:1;00c764f7c3aa:44363 {}] hbase.ChoreService(168): Chore ScheduledChore name=00c764f7c3aa,44363,1733449537002-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-06T01:45:38,336 INFO [RS:1;00c764f7c3aa:44363 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-12-06T01:45:38,336 INFO [RS:1;00c764f7c3aa:44363 {}] regionserver.Replication(171): 00c764f7c3aa,44363,1733449537002 started 2024-12-06T01:45:38,352 INFO [RS:0;00c764f7c3aa:37845 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-12-06T01:45:38,353 INFO [RS:0;00c764f7c3aa:37845 {}] regionserver.HRegionServer(1482): Serving as 00c764f7c3aa,37845,1733449536857, RpcServer on 00c764f7c3aa/172.17.0.2:37845, sessionid=0x100650089b40001 2024-12-06T01:45:38,354 DEBUG [RS:0;00c764f7c3aa:37845 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-12-06T01:45:38,354 DEBUG [RS:0;00c764f7c3aa:37845 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 00c764f7c3aa,37845,1733449536857 2024-12-06T01:45:38,354 DEBUG [RS:0;00c764f7c3aa:37845 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '00c764f7c3aa,37845,1733449536857' 2024-12-06T01:45:38,354 DEBUG [RS:0;00c764f7c3aa:37845 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-12-06T01:45:38,355 DEBUG [RS:0;00c764f7c3aa:37845 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-12-06T01:45:38,356 DEBUG [RS:0;00c764f7c3aa:37845 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-12-06T01:45:38,356 DEBUG [RS:0;00c764f7c3aa:37845 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-12-06T01:45:38,356 DEBUG [RS:0;00c764f7c3aa:37845 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 00c764f7c3aa,37845,1733449536857 2024-12-06T01:45:38,356 DEBUG [RS:0;00c764f7c3aa:37845 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '00c764f7c3aa,37845,1733449536857' 2024-12-06T01:45:38,356 DEBUG [RS:0;00c764f7c3aa:37845 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-12-06T01:45:38,357 DEBUG [RS:0;00c764f7c3aa:37845 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-12-06T01:45:38,358 DEBUG [RS:0;00c764f7c3aa:37845 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-12-06T01:45:38,358 INFO [RS:0;00c764f7c3aa:37845 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-12-06T01:45:38,358 INFO [RS:0;00c764f7c3aa:37845 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-12-06T01:45:38,360 WARN [00c764f7c3aa:40497 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-12-06T01:45:38,361 INFO [RS:1;00c764f7c3aa:44363 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-12-06T01:45:38,362 INFO [RS:1;00c764f7c3aa:44363 {}] regionserver.HRegionServer(1482): Serving as 00c764f7c3aa,44363,1733449537002, RpcServer on 00c764f7c3aa/172.17.0.2:44363, sessionid=0x100650089b40002 2024-12-06T01:45:38,362 DEBUG [RS:1;00c764f7c3aa:44363 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-12-06T01:45:38,362 DEBUG [RS:1;00c764f7c3aa:44363 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 00c764f7c3aa,44363,1733449537002 2024-12-06T01:45:38,362 DEBUG [RS:1;00c764f7c3aa:44363 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '00c764f7c3aa,44363,1733449537002' 2024-12-06T01:45:38,362 DEBUG [RS:1;00c764f7c3aa:44363 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-12-06T01:45:38,363 DEBUG [RS:1;00c764f7c3aa:44363 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-12-06T01:45:38,364 DEBUG [RS:1;00c764f7c3aa:44363 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-12-06T01:45:38,364 DEBUG [RS:1;00c764f7c3aa:44363 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-12-06T01:45:38,364 DEBUG [RS:1;00c764f7c3aa:44363 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 00c764f7c3aa,44363,1733449537002 2024-12-06T01:45:38,364 DEBUG [RS:1;00c764f7c3aa:44363 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '00c764f7c3aa,44363,1733449537002' 2024-12-06T01:45:38,364 DEBUG [RS:1;00c764f7c3aa:44363 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-12-06T01:45:38,365 DEBUG [RS:1;00c764f7c3aa:44363 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-12-06T01:45:38,366 DEBUG [RS:1;00c764f7c3aa:44363 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-12-06T01:45:38,366 INFO [RS:1;00c764f7c3aa:44363 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-12-06T01:45:38,366 INFO [RS:1;00c764f7c3aa:44363 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-12-06T01:45:38,466 INFO [RS:0;00c764f7c3aa:37845 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-06T01:45:38,467 INFO [RS:0;00c764f7c3aa:37845 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=00c764f7c3aa%2C37845%2C1733449536857, suffix=, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/eff72b7a-3322-3fdb-88e1-7287cd98d0d4/e93bd390-0caa-6c00-f03a-52237a196617/WALs/00c764f7c3aa,37845,1733449536857, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/eff72b7a-3322-3fdb-88e1-7287cd98d0d4/e93bd390-0caa-6c00-f03a-52237a196617/oldWALs, maxLogs=55 2024-12-06T01:45:38,467 INFO [RS:1;00c764f7c3aa:44363 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-06T01:45:38,468 INFO [RS:1;00c764f7c3aa:44363 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=00c764f7c3aa%2C44363%2C1733449537002, suffix=, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/eff72b7a-3322-3fdb-88e1-7287cd98d0d4/e93bd390-0caa-6c00-f03a-52237a196617/WALs/00c764f7c3aa,44363,1733449537002, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/eff72b7a-3322-3fdb-88e1-7287cd98d0d4/e93bd390-0caa-6c00-f03a-52237a196617/oldWALs, maxLogs=55 2024-12-06T01:45:38,490 INFO [RS:1;00c764f7c3aa:44363 {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/eff72b7a-3322-3fdb-88e1-7287cd98d0d4/e93bd390-0caa-6c00-f03a-52237a196617/WALs/00c764f7c3aa,44363,1733449537002/00c764f7c3aa%2C44363%2C1733449537002.1733449538471 2024-12-06T01:45:38,491 DEBUG [RS:1;00c764f7c3aa:44363 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-12-06T01:45:38,498 INFO [RS:0;00c764f7c3aa:37845 {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/eff72b7a-3322-3fdb-88e1-7287cd98d0d4/e93bd390-0caa-6c00-f03a-52237a196617/WALs/00c764f7c3aa,37845,1733449536857/00c764f7c3aa%2C37845%2C1733449536857.1733449538471 2024-12-06T01:45:38,498 DEBUG [RS:0;00c764f7c3aa:37845 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-12-06T01:45:38,614 DEBUG [00c764f7c3aa:40497 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=2, allServersCount=2 2024-12-06T01:45:38,629 DEBUG [00c764f7c3aa:40497 {}] balancer.BalancerClusterState(204): Hosts are {00c764f7c3aa=0} racks are {/default-rack=0} 2024-12-06T01:45:38,634 DEBUG [00c764f7c3aa:40497 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-12-06T01:45:38,634 DEBUG [00c764f7c3aa:40497 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-12-06T01:45:38,634 DEBUG [00c764f7c3aa:40497 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-12-06T01:45:38,634 DEBUG [00c764f7c3aa:40497 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-12-06T01:45:38,634 INFO [00c764f7c3aa:40497 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-12-06T01:45:38,634 INFO [00c764f7c3aa:40497 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-12-06T01:45:38,634 DEBUG [00c764f7c3aa:40497 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-12-06T01:45:38,643 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=00c764f7c3aa,44363,1733449537002 2024-12-06T01:45:38,649 INFO [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 00c764f7c3aa,44363,1733449537002, state=OPENING 2024-12-06T01:45:38,654 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2024-12-06T01:45:38,656 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37845-0x100650089b40001, quorum=127.0.0.1:54114, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-06T01:45:38,656 DEBUG [pool-21-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44363-0x100650089b40002, quorum=127.0.0.1:54114, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-06T01:45:38,656 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40497-0x100650089b40000, quorum=127.0.0.1:54114, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-06T01:45:38,657 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-06T01:45:38,657 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-06T01:45:38,657 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-06T01:45:38,659 DEBUG [PEWorker-3 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-12-06T01:45:38,662 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=00c764f7c3aa,44363,1733449537002}] 2024-12-06T01:45:38,837 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-12-06T01:45:38,841 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-4-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38005, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-12-06T01:45:38,850 INFO [RS_OPEN_META-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-12-06T01:45:38,850 INFO [RS_OPEN_META-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-06T01:45:38,851 INFO [RS_OPEN_META-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-12-06T01:45:38,853 INFO [RS_OPEN_META-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=00c764f7c3aa%2C44363%2C1733449537002.meta, suffix=.meta, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/eff72b7a-3322-3fdb-88e1-7287cd98d0d4/e93bd390-0caa-6c00-f03a-52237a196617/WALs/00c764f7c3aa,44363,1733449537002, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/eff72b7a-3322-3fdb-88e1-7287cd98d0d4/e93bd390-0caa-6c00-f03a-52237a196617/oldWALs, maxLogs=55 2024-12-06T01:45:38,874 INFO [RS_OPEN_META-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/eff72b7a-3322-3fdb-88e1-7287cd98d0d4/e93bd390-0caa-6c00-f03a-52237a196617/WALs/00c764f7c3aa,44363,1733449537002/00c764f7c3aa%2C44363%2C1733449537002.meta.1733449538854.meta 2024-12-06T01:45:38,874 DEBUG [RS_OPEN_META-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-12-06T01:45:38,875 DEBUG [RS_OPEN_META-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-12-06T01:45:38,877 DEBUG [RS_OPEN_META-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-12-06T01:45:38,880 DEBUG [RS_OPEN_META-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-12-06T01:45:38,885 INFO [RS_OPEN_META-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-12-06T01:45:38,890 DEBUG [RS_OPEN_META-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-12-06T01:45:38,890 DEBUG [RS_OPEN_META-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-06T01:45:38,891 DEBUG [RS_OPEN_META-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-12-06T01:45:38,891 DEBUG [RS_OPEN_META-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-12-06T01:45:38,892 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family info of region 1588230740 2024-12-06T01:45:38,893 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-12-06T01:45:38,893 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T01:45:38,894 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-06T01:45:38,894 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family ns of region 1588230740 2024-12-06T01:45:38,895 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-12-06T01:45:38,895 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T01:45:38,895 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-06T01:45:38,895 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family rep_barrier of region 1588230740 2024-12-06T01:45:38,896 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-12-06T01:45:38,896 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T01:45:38,896 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-06T01:45:38,896 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family table of region 1588230740 2024-12-06T01:45:38,897 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-12-06T01:45:38,897 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T01:45:38,898 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-06T01:45:38,898 DEBUG [RS_OPEN_META-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-12-06T01:45:38,899 DEBUG [RS_OPEN_META-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/eff72b7a-3322-3fdb-88e1-7287cd98d0d4/e93bd390-0caa-6c00-f03a-52237a196617/data/hbase/meta/1588230740 2024-12-06T01:45:38,900 DEBUG [RS_OPEN_META-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/eff72b7a-3322-3fdb-88e1-7287cd98d0d4/e93bd390-0caa-6c00-f03a-52237a196617/data/hbase/meta/1588230740 2024-12-06T01:45:38,901 DEBUG [RS_OPEN_META-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-12-06T01:45:38,901 DEBUG [RS_OPEN_META-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-12-06T01:45:38,901 DEBUG [RS_OPEN_META-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-12-06T01:45:38,903 DEBUG [RS_OPEN_META-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-12-06T01:45:38,904 INFO [RS_OPEN_META-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=66268454, jitterRate=-0.012523084878921509}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-12-06T01:45:38,904 DEBUG [RS_OPEN_META-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-12-06T01:45:38,906 DEBUG [RS_OPEN_META-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1733449538891Writing region info on filesystem at 1733449538891Initializing all the Stores at 1733449538892 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733449538892Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733449538892Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733449538892Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733449538892Cleaning up temporary data from old regions at 1733449538901 (+9 ms)Running coprocessor post-open hooks at 1733449538904 (+3 ms)Region opened successfully at 1733449538906 (+2 ms) 2024-12-06T01:45:38,914 INFO [RS_OPEN_META-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1733449538828 2024-12-06T01:45:38,928 DEBUG [RS_OPEN_META-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-12-06T01:45:38,928 INFO [RS_OPEN_META-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-12-06T01:45:38,929 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=00c764f7c3aa,44363,1733449537002 2024-12-06T01:45:38,930 INFO [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 00c764f7c3aa,44363,1733449537002, state=OPEN 2024-12-06T01:45:38,933 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37845-0x100650089b40001, quorum=127.0.0.1:54114, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-06T01:45:38,933 DEBUG [pool-21-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44363-0x100650089b40002, quorum=127.0.0.1:54114, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-06T01:45:38,933 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40497-0x100650089b40000, quorum=127.0.0.1:54114, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-06T01:45:38,934 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-06T01:45:38,934 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-06T01:45:38,934 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-06T01:45:38,934 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=00c764f7c3aa,44363,1733449537002 2024-12-06T01:45:38,937 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2 2024-12-06T01:45:38,937 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=00c764f7c3aa,44363,1733449537002 in 273 msec 2024-12-06T01:45:38,942 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1 2024-12-06T01:45:38,942 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 742 msec 2024-12-06T01:45:38,943 DEBUG [PEWorker-2 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta 2024-12-06T01:45:38,943 INFO [PEWorker-2 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces 2024-12-06T01:45:38,965 DEBUG [PEWorker-2 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-06T01:45:38,966 DEBUG [PEWorker-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=00c764f7c3aa,44363,1733449537002, seqNum=-1] 2024-12-06T01:45:38,987 DEBUG [PEWorker-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-06T01:45:38,990 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-4-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50451, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-06T01:45:39,011 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 1.0220 sec 2024-12-06T01:45:39,011 INFO [master/00c764f7c3aa:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1733449539011, completionTime=-1 2024-12-06T01:45:39,014 INFO [master/00c764f7c3aa:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=2; waited=0ms, expected min=2 server(s), max=2 server(s), master is running 2024-12-06T01:45:39,014 DEBUG [master/00c764f7c3aa:0:becomeActiveMaster {}] assignment.AssignmentManager(1764): Joining cluster... 2024-12-06T01:45:39,047 INFO [master/00c764f7c3aa:0:becomeActiveMaster {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-12-06T01:45:39,048 INFO [master/00c764f7c3aa:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1733449599047 2024-12-06T01:45:39,048 INFO [master/00c764f7c3aa:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1733449659048 2024-12-06T01:45:39,048 INFO [master/00c764f7c3aa:0:becomeActiveMaster {}] assignment.AssignmentManager(1783): Joined the cluster in 33 msec 2024-12-06T01:45:39,050 DEBUG [master/00c764f7c3aa:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 1588230740 changed from -1.0 to 0.0, refreshing cache 2024-12-06T01:45:39,057 INFO [master/00c764f7c3aa:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=00c764f7c3aa,40497,1733449536009-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-06T01:45:39,057 INFO [master/00c764f7c3aa:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=00c764f7c3aa,40497,1733449536009-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2024-12-06T01:45:39,057 INFO [master/00c764f7c3aa:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=00c764f7c3aa,40497,1733449536009-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2024-12-06T01:45:39,059 INFO [master/00c764f7c3aa:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-00c764f7c3aa:40497, period=300000, unit=MILLISECONDS is enabled. 2024-12-06T01:45:39,060 INFO [master/00c764f7c3aa:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2024-12-06T01:45:39,060 INFO [master/00c764f7c3aa:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled. 2024-12-06T01:45:39,067 DEBUG [master/00c764f7c3aa:0.Chore.1 {}] janitor.CatalogJanitor(180): 2024-12-06T01:45:39,091 INFO [master/00c764f7c3aa:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 1.964sec 2024-12-06T01:45:39,093 INFO [master/00c764f7c3aa:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2024-12-06T01:45:39,094 INFO [master/00c764f7c3aa:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2024-12-06T01:45:39,095 INFO [master/00c764f7c3aa:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2024-12-06T01:45:39,096 INFO [master/00c764f7c3aa:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2024-12-06T01:45:39,096 INFO [master/00c764f7c3aa:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2024-12-06T01:45:39,097 INFO [master/00c764f7c3aa:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=00c764f7c3aa,40497,1733449536009-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-12-06T01:45:39,097 INFO [master/00c764f7c3aa:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=00c764f7c3aa,40497,1733449536009-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2024-12-06T01:45:39,099 DEBUG [master/00c764f7c3aa:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds 2024-12-06T01:45:39,100 INFO [master/00c764f7c3aa:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2024-12-06T01:45:39,100 INFO [master/00c764f7c3aa:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=00c764f7c3aa,40497,1733449536009-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled. 2024-12-06T01:45:39,162 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@73434faf, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-06T01:45:39,165 DEBUG [Time-limited test {}] nio.NioEventLoop(110): -Dio.netty.noKeySetOptimization: false 2024-12-06T01:45:39,166 DEBUG [Time-limited test {}] nio.NioEventLoop(111): -Dio.netty.selectorAutoRebuildThreshold: 512 2024-12-06T01:45:39,171 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 00c764f7c3aa,40497,-1 for getting cluster id 2024-12-06T01:45:39,173 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-06T01:45:39,182 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '0649e6ab-29a8-4257-8352-d4d974c2a6de' 2024-12-06T01:45:39,184 DEBUG [RPCClient-NioEventLoopGroup-5-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-06T01:45:39,185 DEBUG [RPCClient-NioEventLoopGroup-5-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "0649e6ab-29a8-4257-8352-d4d974c2a6de" 2024-12-06T01:45:39,185 DEBUG [RPCClient-NioEventLoopGroup-5-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6ef386b6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-06T01:45:39,185 DEBUG [RPCClient-NioEventLoopGroup-5-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [00c764f7c3aa,40497,-1] 2024-12-06T01:45:39,188 DEBUG [RPCClient-NioEventLoopGroup-5-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-06T01:45:39,190 DEBUG [RPCClient-NioEventLoopGroup-5-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-06T01:45:39,191 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51960, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-06T01:45:39,194 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@70a596f9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-06T01:45:39,194 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-06T01:45:39,203 DEBUG [RPCClient-NioEventLoopGroup-5-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=00c764f7c3aa,44363,1733449537002, seqNum=-1] 2024-12-06T01:45:39,204 DEBUG [RPCClient-NioEventLoopGroup-5-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-06T01:45:39,207 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-4-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54324, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-06T01:45:39,227 INFO [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=00c764f7c3aa,40497,1733449536009 2024-12-06T01:45:39,231 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-12-06T01:45:39,236 DEBUG [RPCClient-NioEventLoopGroup-5-2 {}] client.AsyncConnectionImpl(321): The fetched master address is 00c764f7c3aa,40497,1733449536009 2024-12-06T01:45:39,256 DEBUG [RPCClient-NioEventLoopGroup-5-2 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@5bd5631f 2024-12-06T01:45:39,257 DEBUG [RPCClient-NioEventLoopGroup-5-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-12-06T01:45:39,262 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51964, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-12-06T01:45:39,270 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40497 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'testBlockEvictionOnRegionMove1733449539227', {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-06T01:45:39,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40497 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=testBlockEvictionOnRegionMove1733449539227 2024-12-06T01:45:39,279 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=testBlockEvictionOnRegionMove1733449539227 execute state=CREATE_TABLE_PRE_OPERATION 2024-12-06T01:45:39,282 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40497 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "testBlockEvictionOnRegionMove1733449539227" procId is: 4 2024-12-06T01:45:39,282 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T01:45:39,284 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=testBlockEvictionOnRegionMove1733449539227 execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-12-06T01:45:39,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40497 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-06T01:45:39,298 INFO [RegionOpenAndInit-testBlockEvictionOnRegionMove1733449539227-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => a1102c9d50dc8624cb9af2bfbf31bc75, NAME => 'testBlockEvictionOnRegionMove1733449539227,,1733449539263.a1102c9d50dc8624cb9af2bfbf31bc75.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testBlockEvictionOnRegionMove1733449539227', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/eff72b7a-3322-3fdb-88e1-7287cd98d0d4/e93bd390-0caa-6c00-f03a-52237a196617 2024-12-06T01:45:39,305 DEBUG [RegionOpenAndInit-testBlockEvictionOnRegionMove1733449539227-pool-0 {}] regionserver.HRegion(898): Instantiated testBlockEvictionOnRegionMove1733449539227,,1733449539263.a1102c9d50dc8624cb9af2bfbf31bc75.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-06T01:45:39,306 DEBUG [RegionOpenAndInit-testBlockEvictionOnRegionMove1733449539227-pool-0 {}] regionserver.HRegion(1722): Closing a1102c9d50dc8624cb9af2bfbf31bc75, disabling compactions & flushes 2024-12-06T01:45:39,306 INFO [RegionOpenAndInit-testBlockEvictionOnRegionMove1733449539227-pool-0 {}] regionserver.HRegion(1755): Closing region testBlockEvictionOnRegionMove1733449539227,,1733449539263.a1102c9d50dc8624cb9af2bfbf31bc75. 2024-12-06T01:45:39,306 DEBUG [RegionOpenAndInit-testBlockEvictionOnRegionMove1733449539227-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on testBlockEvictionOnRegionMove1733449539227,,1733449539263.a1102c9d50dc8624cb9af2bfbf31bc75. 2024-12-06T01:45:39,306 DEBUG [RegionOpenAndInit-testBlockEvictionOnRegionMove1733449539227-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on testBlockEvictionOnRegionMove1733449539227,,1733449539263.a1102c9d50dc8624cb9af2bfbf31bc75. after waiting 0 ms 2024-12-06T01:45:39,306 DEBUG [RegionOpenAndInit-testBlockEvictionOnRegionMove1733449539227-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region testBlockEvictionOnRegionMove1733449539227,,1733449539263.a1102c9d50dc8624cb9af2bfbf31bc75. 2024-12-06T01:45:39,306 INFO [RegionOpenAndInit-testBlockEvictionOnRegionMove1733449539227-pool-0 {}] regionserver.HRegion(1973): Closed testBlockEvictionOnRegionMove1733449539227,,1733449539263.a1102c9d50dc8624cb9af2bfbf31bc75. 2024-12-06T01:45:39,306 DEBUG [RegionOpenAndInit-testBlockEvictionOnRegionMove1733449539227-pool-0 {}] regionserver.HRegion(1676): Region close journal for a1102c9d50dc8624cb9af2bfbf31bc75: Waiting for close lock at 1733449539306Disabling compacts and flushes for region at 1733449539306Disabling writes for close at 1733449539306Writing region close event to WAL at 1733449539306Closed at 1733449539306 2024-12-06T01:45:39,308 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=testBlockEvictionOnRegionMove1733449539227 execute state=CREATE_TABLE_ADD_TO_META 2024-12-06T01:45:39,313 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testBlockEvictionOnRegionMove1733449539227,,1733449539263.a1102c9d50dc8624cb9af2bfbf31bc75.","families":{"info":[{"qualifier":"regioninfo","vlen":76,"tag":[],"timestamp":"1733449539308"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733449539308"}]},"ts":"1733449539308"} 2024-12-06T01:45:39,318 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-12-06T01:45:39,319 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=testBlockEvictionOnRegionMove1733449539227 execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-12-06T01:45:39,321 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testBlockEvictionOnRegionMove1733449539227","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733449539319"}]},"ts":"1733449539319"} 2024-12-06T01:45:39,325 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=testBlockEvictionOnRegionMove1733449539227, state=ENABLING in hbase:meta 2024-12-06T01:45:39,326 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(204): Hosts are {00c764f7c3aa=0} racks are {/default-rack=0} 2024-12-06T01:45:39,327 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-12-06T01:45:39,327 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-12-06T01:45:39,327 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-12-06T01:45:39,327 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-12-06T01:45:39,327 INFO [PEWorker-3 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-12-06T01:45:39,327 INFO [PEWorker-3 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-12-06T01:45:39,327 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-12-06T01:45:39,329 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1733449539227, region=a1102c9d50dc8624cb9af2bfbf31bc75, ASSIGN}] 2024-12-06T01:45:39,331 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1733449539227, region=a1102c9d50dc8624cb9af2bfbf31bc75, ASSIGN 2024-12-06T01:45:39,332 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1733449539227, region=a1102c9d50dc8624cb9af2bfbf31bc75, ASSIGN; state=OFFLINE, location=00c764f7c3aa,37845,1733449536857; forceNewPlan=false, retain=false 2024-12-06T01:45:39,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40497 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-06T01:45:39,484 INFO [00c764f7c3aa:40497 {}] balancer.BaseLoadBalancer(388): Reassigned 1 regions. 1 retained the pre-restart assignment. 2024-12-06T01:45:39,485 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=a1102c9d50dc8624cb9af2bfbf31bc75, regionState=OPENING, regionLocation=00c764f7c3aa,37845,1733449536857 2024-12-06T01:45:39,490 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-4-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1733449539227, region=a1102c9d50dc8624cb9af2bfbf31bc75, ASSIGN because future has completed 2024-12-06T01:45:39,491 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure a1102c9d50dc8624cb9af2bfbf31bc75, server=00c764f7c3aa,37845,1733449536857}] 2024-12-06T01:45:39,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40497 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-06T01:45:39,644 DEBUG [RSProcedureDispatcher-pool-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-12-06T01:45:39,646 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46249, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-12-06T01:45:39,649 INFO [RS_OPEN_REGION-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(132): Open testBlockEvictionOnRegionMove1733449539227,,1733449539263.a1102c9d50dc8624cb9af2bfbf31bc75. 2024-12-06T01:45:39,649 DEBUG [RS_OPEN_REGION-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7752): Opening region: {ENCODED => a1102c9d50dc8624cb9af2bfbf31bc75, NAME => 'testBlockEvictionOnRegionMove1733449539227,,1733449539263.a1102c9d50dc8624cb9af2bfbf31bc75.', STARTKEY => '', ENDKEY => ''} 2024-12-06T01:45:39,649 DEBUG [RS_OPEN_REGION-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testBlockEvictionOnRegionMove1733449539227 a1102c9d50dc8624cb9af2bfbf31bc75 2024-12-06T01:45:39,650 DEBUG [RS_OPEN_REGION-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(898): Instantiated testBlockEvictionOnRegionMove1733449539227,,1733449539263.a1102c9d50dc8624cb9af2bfbf31bc75.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-06T01:45:39,650 DEBUG [RS_OPEN_REGION-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7794): checking encryption for a1102c9d50dc8624cb9af2bfbf31bc75 2024-12-06T01:45:39,650 DEBUG [RS_OPEN_REGION-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7797): checking classloading for a1102c9d50dc8624cb9af2bfbf31bc75 2024-12-06T01:45:39,651 INFO [StoreOpener-a1102c9d50dc8624cb9af2bfbf31bc75-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family family of region a1102c9d50dc8624cb9af2bfbf31bc75 2024-12-06T01:45:39,656 INFO [StoreOpener-a1102c9d50dc8624cb9af2bfbf31bc75-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region a1102c9d50dc8624cb9af2bfbf31bc75 columnFamilyName family 2024-12-06T01:45:39,657 DEBUG [StoreOpener-a1102c9d50dc8624cb9af2bfbf31bc75-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T01:45:39,657 INFO [StoreOpener-a1102c9d50dc8624cb9af2bfbf31bc75-1 {}] regionserver.HStore(327): Store=a1102c9d50dc8624cb9af2bfbf31bc75/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-06T01:45:39,657 DEBUG [RS_OPEN_REGION-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1038): replaying wal for a1102c9d50dc8624cb9af2bfbf31bc75 2024-12-06T01:45:39,658 DEBUG [RS_OPEN_REGION-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/eff72b7a-3322-3fdb-88e1-7287cd98d0d4/e93bd390-0caa-6c00-f03a-52237a196617/data/default/testBlockEvictionOnRegionMove1733449539227/a1102c9d50dc8624cb9af2bfbf31bc75 2024-12-06T01:45:39,658 DEBUG [RS_OPEN_REGION-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/eff72b7a-3322-3fdb-88e1-7287cd98d0d4/e93bd390-0caa-6c00-f03a-52237a196617/data/default/testBlockEvictionOnRegionMove1733449539227/a1102c9d50dc8624cb9af2bfbf31bc75 2024-12-06T01:45:39,659 DEBUG [RS_OPEN_REGION-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1048): stopping wal replay for a1102c9d50dc8624cb9af2bfbf31bc75 2024-12-06T01:45:39,659 DEBUG [RS_OPEN_REGION-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1060): Cleaning up temporary data for a1102c9d50dc8624cb9af2bfbf31bc75 2024-12-06T01:45:39,659 DEBUG [RS_OPEN_REGION-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1093): writing seq id for a1102c9d50dc8624cb9af2bfbf31bc75 2024-12-06T01:45:39,666 DEBUG [RS_OPEN_REGION-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/eff72b7a-3322-3fdb-88e1-7287cd98d0d4/e93bd390-0caa-6c00-f03a-52237a196617/data/default/testBlockEvictionOnRegionMove1733449539227/a1102c9d50dc8624cb9af2bfbf31bc75/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-06T01:45:39,667 INFO [RS_OPEN_REGION-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1114): Opened a1102c9d50dc8624cb9af2bfbf31bc75; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=70521014, jitterRate=0.05084499716758728}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-06T01:45:39,667 DEBUG [RS_OPEN_REGION-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1122): Running coprocessor post-open hooks for a1102c9d50dc8624cb9af2bfbf31bc75 2024-12-06T01:45:39,668 DEBUG [RS_OPEN_REGION-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1006): Region open journal for a1102c9d50dc8624cb9af2bfbf31bc75: Running coprocessor pre-open hook at 1733449539650Writing region info on filesystem at 1733449539650Initializing all the Stores at 1733449539650Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733449539650Cleaning up temporary data from old regions at 1733449539659 (+9 ms)Running coprocessor post-open hooks at 1733449539667 (+8 ms)Region opened successfully at 1733449539668 (+1 ms) 2024-12-06T01:45:39,669 INFO [RS_OPEN_REGION-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2236): Post open deploy tasks for testBlockEvictionOnRegionMove1733449539227,,1733449539263.a1102c9d50dc8624cb9af2bfbf31bc75., pid=6, masterSystemTime=1733449539643 2024-12-06T01:45:39,673 DEBUG [RS_OPEN_REGION-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2266): Finished post open deploy task for testBlockEvictionOnRegionMove1733449539227,,1733449539263.a1102c9d50dc8624cb9af2bfbf31bc75. 2024-12-06T01:45:39,673 INFO [RS_OPEN_REGION-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(153): Opened testBlockEvictionOnRegionMove1733449539227,,1733449539263.a1102c9d50dc8624cb9af2bfbf31bc75. 2024-12-06T01:45:39,674 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=a1102c9d50dc8624cb9af2bfbf31bc75, regionState=OPEN, openSeqNum=2, regionLocation=00c764f7c3aa,37845,1733449536857 2024-12-06T01:45:39,677 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-4-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure a1102c9d50dc8624cb9af2bfbf31bc75, server=00c764f7c3aa,37845,1733449536857 because future has completed 2024-12-06T01:45:39,681 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=6, resume processing ppid=5 2024-12-06T01:45:39,681 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure a1102c9d50dc8624cb9af2bfbf31bc75, server=00c764f7c3aa,37845,1733449536857 in 188 msec 2024-12-06T01:45:39,683 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=5, resume processing ppid=4 2024-12-06T01:45:39,684 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1733449539227, region=a1102c9d50dc8624cb9af2bfbf31bc75, ASSIGN in 352 msec 2024-12-06T01:45:39,684 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=testBlockEvictionOnRegionMove1733449539227 execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-12-06T01:45:39,685 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testBlockEvictionOnRegionMove1733449539227","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733449539684"}]},"ts":"1733449539684"} 2024-12-06T01:45:39,687 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=testBlockEvictionOnRegionMove1733449539227, state=ENABLED in hbase:meta 2024-12-06T01:45:39,688 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=testBlockEvictionOnRegionMove1733449539227 execute state=CREATE_TABLE_POST_OPERATION 2024-12-06T01:45:39,690 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=testBlockEvictionOnRegionMove1733449539227 in 415 msec 2024-12-06T01:45:39,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40497 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-06T01:45:39,920 INFO [RPCClient-NioEventLoopGroup-5-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:testBlockEvictionOnRegionMove1733449539227 completed 2024-12-06T01:45:39,920 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table testBlockEvictionOnRegionMove1733449539227 get assigned. Timeout = 60000ms 2024-12-06T01:45:39,924 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-12-06T01:45:39,930 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table testBlockEvictionOnRegionMove1733449539227 assigned to meta. Checking AM states. 2024-12-06T01:45:39,930 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-12-06T01:45:39,931 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table testBlockEvictionOnRegionMove1733449539227 assigned. 2024-12-06T01:45:39,941 DEBUG [RPCClient-NioEventLoopGroup-5-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testBlockEvictionOnRegionMove1733449539227', row='row1', locateType=CURRENT is [region=testBlockEvictionOnRegionMove1733449539227,,1733449539263.a1102c9d50dc8624cb9af2bfbf31bc75., hostname=00c764f7c3aa,37845,1733449536857, seqNum=2] 2024-12-06T01:45:39,942 DEBUG [RPCClient-NioEventLoopGroup-5-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-06T01:45:39,945 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36888, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-06T01:45:39,956 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing a1102c9d50dc8624cb9af2bfbf31bc75 1/1 column families, dataSize=78 B heapSize=480 B 2024-12-06T01:45:39,983 DEBUG [Time-limited test {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 6633fe18a90f4711b6189f0d51fa54a8 2024-12-06T01:45:40,028 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/eff72b7a-3322-3fdb-88e1-7287cd98d0d4/e93bd390-0caa-6c00-f03a-52237a196617/data/default/testBlockEvictionOnRegionMove1733449539227/a1102c9d50dc8624cb9af2bfbf31bc75/.tmp/family/6633fe18a90f4711b6189f0d51fa54a8 is 43, key is row1/family:qf1/1/Put/seqid=0 2024-12-06T01:45:40,035 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=78 B at sequenceid=6 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/eff72b7a-3322-3fdb-88e1-7287cd98d0d4/e93bd390-0caa-6c00-f03a-52237a196617/data/default/testBlockEvictionOnRegionMove1733449539227/a1102c9d50dc8624cb9af2bfbf31bc75/.tmp/family/6633fe18a90f4711b6189f0d51fa54a8 2024-12-06T01:45:40,074 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/eff72b7a-3322-3fdb-88e1-7287cd98d0d4/e93bd390-0caa-6c00-f03a-52237a196617/data/default/testBlockEvictionOnRegionMove1733449539227/a1102c9d50dc8624cb9af2bfbf31bc75/.tmp/family/6633fe18a90f4711b6189f0d51fa54a8 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/eff72b7a-3322-3fdb-88e1-7287cd98d0d4/e93bd390-0caa-6c00-f03a-52237a196617/data/default/testBlockEvictionOnRegionMove1733449539227/a1102c9d50dc8624cb9af2bfbf31bc75/family/6633fe18a90f4711b6189f0d51fa54a8 2024-12-06T01:45:40,075 DEBUG [Time-limited test {}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/eff72b7a-3322-3fdb-88e1-7287cd98d0d4/e93bd390-0caa-6c00-f03a-52237a196617/data/default/testBlockEvictionOnRegionMove1733449539227/a1102c9d50dc8624cb9af2bfbf31bc75/family/6633fe18a90f4711b6189f0d51fa54a8, delay=946 ms 2024-12-06T01:45:40,077 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/eff72b7a-3322-3fdb-88e1-7287cd98d0d4/e93bd390-0caa-6c00-f03a-52237a196617/data/default/testBlockEvictionOnRegionMove1733449539227/a1102c9d50dc8624cb9af2bfbf31bc75/family/6633fe18a90f4711b6189f0d51fa54a8, entries=2, sequenceid=6, filesize=4.9 K 2024-12-06T01:45:40,083 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~78 B/78, heapSize ~464 B/464, currentSize=0 B/0 for a1102c9d50dc8624cb9af2bfbf31bc75 in 127ms, sequenceid=6, compaction requested=false 2024-12-06T01:45:40,084 DEBUG [Time-limited test {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'testBlockEvictionOnRegionMove1733449539227' 2024-12-06T01:45:40,086 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for a1102c9d50dc8624cb9af2bfbf31bc75: 2024-12-06T01:45:41,024 DEBUG [hfile-prefetch-1733449540075 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 6633fe18a90f4711b6189f0d51fa54a8_0. 2024-12-06T01:45:41,030 DEBUG [hfile-prefetch-1733449540075 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 6633fe18a90f4711b6189f0d51fa54a8_117. 2024-12-06T01:45:41,032 DEBUG [hfile-prefetch-1733449540075 {}] bucket.BucketCache(2281): Notifying caching completed for file file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/eff72b7a-3322-3fdb-88e1-7287cd98d0d4/e93bd390-0caa-6c00-f03a-52237a196617/data/default/testBlockEvictionOnRegionMove1733449539227/a1102c9d50dc8624cb9af2bfbf31bc75/family/6633fe18a90f4711b6189f0d51fa54a8, with total blocks 2, and data blocks 1 2024-12-06T01:45:41,032 DEBUG [hfile-prefetch-1733449540075 {}] bucket.BucketCache(2285): iterating over 0 entries in the backing map 2024-12-06T01:45:41,033 DEBUG [hfile-prefetch-1733449540075 {}] bucket.BucketCache(2308): Prefetch executor completed for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/eff72b7a-3322-3fdb-88e1-7287cd98d0d4/e93bd390-0caa-6c00-f03a-52237a196617/data/default/testBlockEvictionOnRegionMove1733449539227/a1102c9d50dc8624cb9af2bfbf31bc75/family/6633fe18a90f4711b6189f0d51fa54a8, but only 0 data blocks were cached. Total data blocks for file: 1. Checking for blocks pending cache in cache writer queue. 2024-12-06T01:45:41,033 DEBUG [hfile-prefetch-1733449540075 {}] bucket.BucketCache(2318): There are still blocks pending caching for file file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/eff72b7a-3322-3fdb-88e1-7287cd98d0d4/e93bd390-0caa-6c00-f03a-52237a196617/data/default/testBlockEvictionOnRegionMove1733449539227/a1102c9d50dc8624cb9af2bfbf31bc75/family/6633fe18a90f4711b6189f0d51fa54a8. Will sleep 100ms and try the verification again. 2024-12-06T01:45:41,056 DEBUG [bucket-cache-persister {}] bucket.BucketCachePersister(47): Cache is inconsistent, persisting to disk 2024-12-06T01:45:41,056 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1487): Thread bucket-cache-persister started persisting bucket cache to file 2024-12-06T01:45:41,057 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1494): Persist in new chunked persistence format. 2024-12-06T01:45:41,057 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1673): persistToFile: before persisting backing map size: 1, fullycachedFiles size: 0, chunkSize: 10000 2024-12-06T01:45:41,080 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1680): persistToFile: after persisting backing map size: 1, fullycachedFiles size: 0 2024-12-06T01:45:41,080 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1498): PersistToFile: after persisting backing map size: 1, fullycachedFiles size: 0, file name: bucket.persistence1733449541056 2024-12-06T01:45:41,080 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1509): Thread bucket-cache-persister finished persisting bucket cache to file, renaming 2024-12-06T01:45:41,097 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40497 {}] master.HMaster(2410): Client=jenkins//172.17.0.2 move hri=a1102c9d50dc8624cb9af2bfbf31bc75, source=00c764f7c3aa,37845,1733449536857, destination=00c764f7c3aa,44363,1733449537002, warming up region on 00c764f7c3aa,44363,1733449537002 2024-12-06T01:45:41,099 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40497 {}] master.HMaster(2414): Client=jenkins//172.17.0.2 move hri=a1102c9d50dc8624cb9af2bfbf31bc75, source=00c764f7c3aa,37845,1733449536857, destination=00c764f7c3aa,44363,1733449537002, running balancer 2024-12-06T01:45:41,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40497 {}] procedure2.ProcedureExecutor(1139): Stored pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1733449539227, region=a1102c9d50dc8624cb9af2bfbf31bc75, REOPEN/MOVE 2024-12-06T01:45:41,101 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1733449539227, region=a1102c9d50dc8624cb9af2bfbf31bc75, REOPEN/MOVE 2024-12-06T01:45:41,101 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44363 {}] regionserver.RSRpcServices(2066): Warmup testBlockEvictionOnRegionMove1733449539227,,1733449539263.a1102c9d50dc8624cb9af2bfbf31bc75. 2024-12-06T01:45:41,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44363 {}] regionserver.HRegion(7855): Warmup {ENCODED => a1102c9d50dc8624cb9af2bfbf31bc75, NAME => 'testBlockEvictionOnRegionMove1733449539227,,1733449539263.a1102c9d50dc8624cb9af2bfbf31bc75.', STARTKEY => '', ENDKEY => ''} 2024-12-06T01:45:41,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44363 {}] regionserver.HRegion(898): Instantiated testBlockEvictionOnRegionMove1733449539227,,1733449539263.a1102c9d50dc8624cb9af2bfbf31bc75.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-06T01:45:41,103 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=a1102c9d50dc8624cb9af2bfbf31bc75, regionState=CLOSING, regionLocation=00c764f7c3aa,37845,1733449536857 2024-12-06T01:45:41,103 INFO [StoreOpener-a1102c9d50dc8624cb9af2bfbf31bc75-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family family of region a1102c9d50dc8624cb9af2bfbf31bc75 2024-12-06T01:45:41,104 INFO [StoreOpener-a1102c9d50dc8624cb9af2bfbf31bc75-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region a1102c9d50dc8624cb9af2bfbf31bc75 columnFamilyName family 2024-12-06T01:45:41,104 DEBUG [StoreOpener-a1102c9d50dc8624cb9af2bfbf31bc75-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T01:45:41,106 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-4-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1733449539227, region=a1102c9d50dc8624cb9af2bfbf31bc75, REOPEN/MOVE because future has completed 2024-12-06T01:45:41,106 DEBUG [PEWorker-3 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: true 2024-12-06T01:45:41,107 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=8, ppid=7, state=RUNNABLE, hasLock=false; CloseRegionProcedure a1102c9d50dc8624cb9af2bfbf31bc75, server=00c764f7c3aa,37845,1733449536857}] 2024-12-06T01:45:41,108 DEBUG [StoreOpener-a1102c9d50dc8624cb9af2bfbf31bc75-1 {}] regionserver.StoreEngine(278): loaded file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/eff72b7a-3322-3fdb-88e1-7287cd98d0d4/e93bd390-0caa-6c00-f03a-52237a196617/data/default/testBlockEvictionOnRegionMove1733449539227/a1102c9d50dc8624cb9af2bfbf31bc75/family/6633fe18a90f4711b6189f0d51fa54a8 2024-12-06T01:45:41,108 INFO [StoreOpener-a1102c9d50dc8624cb9af2bfbf31bc75-1 {}] regionserver.HStore(327): Store=a1102c9d50dc8624cb9af2bfbf31bc75/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-06T01:45:41,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44363 {}] regionserver.HRegion(1722): Closing a1102c9d50dc8624cb9af2bfbf31bc75, disabling compactions & flushes 2024-12-06T01:45:41,109 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44363 {}] regionserver.HRegion(1755): Closing region testBlockEvictionOnRegionMove1733449539227,,1733449539263.a1102c9d50dc8624cb9af2bfbf31bc75. 2024-12-06T01:45:41,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44363 {}] regionserver.HRegion(1776): Time limited wait for close lock on testBlockEvictionOnRegionMove1733449539227,,1733449539263.a1102c9d50dc8624cb9af2bfbf31bc75. 2024-12-06T01:45:41,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44363 {}] regionserver.HRegion(1843): Acquired close lock on testBlockEvictionOnRegionMove1733449539227,,1733449539263.a1102c9d50dc8624cb9af2bfbf31bc75. after waiting 0 ms 2024-12-06T01:45:41,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44363 {}] regionserver.HRegion(1853): Updates disabled for region testBlockEvictionOnRegionMove1733449539227,,1733449539263.a1102c9d50dc8624cb9af2bfbf31bc75. 2024-12-06T01:45:41,110 DEBUG [StoreFileCloser-a1102c9d50dc8624cb9af2bfbf31bc75-family-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/eff72b7a-3322-3fdb-88e1-7287cd98d0d4/e93bd390-0caa-6c00-f03a-52237a196617/data/default/testBlockEvictionOnRegionMove1733449539227/a1102c9d50dc8624cb9af2bfbf31bc75/family/6633fe18a90f4711b6189f0d51fa54a8 2024-12-06T01:45:41,110 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44363 {}] regionserver.HRegion(1973): Closed testBlockEvictionOnRegionMove1733449539227,,1733449539263.a1102c9d50dc8624cb9af2bfbf31bc75. 2024-12-06T01:45:41,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44363 {}] regionserver.HRegion(1676): Region close journal for a1102c9d50dc8624cb9af2bfbf31bc75: Waiting for close lock at 1733449541109Disabling compacts and flushes for region at 1733449541109Disabling writes for close at 1733449541109Writing region close event to WAL at 1733449541110 (+1 ms)Closed at 1733449541110 2024-12-06T01:45:41,113 WARN [hfile-prefetch-1733449540075 {}] hfile.HFilePreadReader$1(158): Prefetch path=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/eff72b7a-3322-3fdb-88e1-7287cd98d0d4/e93bd390-0caa-6c00-f03a-52237a196617/data/default/testBlockEvictionOnRegionMove1733449539227/a1102c9d50dc8624cb9af2bfbf31bc75/family/6633fe18a90f4711b6189f0d51fa54a8, offset=158, end=158 java.lang.RuntimeException: java.lang.InterruptedException: sleep interrupted at org.apache.hadoop.hbase.io.hfile.bucket.BucketCache.notifyFileCachingCompleted(BucketCache.java:2337) ~[classes/:?] at org.apache.hadoop.hbase.io.hfile.CombinedBlockCache.notifyFileCachingCompleted(CombinedBlockCache.java:469) ~[classes/:?] at org.apache.hadoop.hbase.io.hfile.HFilePreadReader$1.run(HFilePreadReader.java:148) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.lambda$tracedRunnable$2(TraceUtil.java:155) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:539) ~[?:?] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:304) ~[?:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCache.notifyFileCachingCompleted(BucketCache.java:2320) ~[classes/:?] ... 9 more 2024-12-06T01:45:41,114 DEBUG [hfile-prefetch-1733449540075 {}] hfile.PrefetchExecutor(121): Prefetch completed for 6633fe18a90f4711b6189f0d51fa54a8 2024-12-06T01:45:41,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40497 {}] procedure.ProcedureSyncWait(219): waitFor pid=7 2024-12-06T01:45:41,265 INFO [RS_CLOSE_REGION-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] handler.UnassignRegionHandler(122): Close a1102c9d50dc8624cb9af2bfbf31bc75 2024-12-06T01:45:41,266 DEBUG [RS_CLOSE_REGION-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: true 2024-12-06T01:45:41,267 DEBUG [RS_CLOSE_REGION-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1722): Closing a1102c9d50dc8624cb9af2bfbf31bc75, disabling compactions & flushes 2024-12-06T01:45:41,267 INFO [RS_CLOSE_REGION-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1755): Closing region testBlockEvictionOnRegionMove1733449539227,,1733449539263.a1102c9d50dc8624cb9af2bfbf31bc75. 2024-12-06T01:45:41,267 DEBUG [RS_CLOSE_REGION-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1776): Time limited wait for close lock on testBlockEvictionOnRegionMove1733449539227,,1733449539263.a1102c9d50dc8624cb9af2bfbf31bc75. 2024-12-06T01:45:41,267 DEBUG [RS_CLOSE_REGION-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1843): Acquired close lock on testBlockEvictionOnRegionMove1733449539227,,1733449539263.a1102c9d50dc8624cb9af2bfbf31bc75. after waiting 0 ms 2024-12-06T01:45:41,267 DEBUG [RS_CLOSE_REGION-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1853): Updates disabled for region testBlockEvictionOnRegionMove1733449539227,,1733449539263.a1102c9d50dc8624cb9af2bfbf31bc75. 2024-12-06T01:45:41,269 DEBUG [StoreFileCloser-a1102c9d50dc8624cb9af2bfbf31bc75-family-1 {}] bucket.BucketCache(1863): found 1 blocks for file 6633fe18a90f4711b6189f0d51fa54a8, starting offset: 0, end offset: 9223372036854775807 2024-12-06T01:45:41,270 DEBUG [StoreFileCloser-a1102c9d50dc8624cb9af2bfbf31bc75-family-1 {}] bucket.BucketCache(856): removed key 6633fe18a90f4711b6189f0d51fa54a8_0 from back map with offset lock 2094080 in the evict process 2024-12-06T01:45:41,276 DEBUG [RS_CLOSE_REGION-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/eff72b7a-3322-3fdb-88e1-7287cd98d0d4/e93bd390-0caa-6c00-f03a-52237a196617/data/default/testBlockEvictionOnRegionMove1733449539227/a1102c9d50dc8624cb9af2bfbf31bc75/recovered.edits/9.seqid, newMaxSeqId=9, maxSeqId=1 2024-12-06T01:45:41,278 INFO [RS_CLOSE_REGION-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1973): Closed testBlockEvictionOnRegionMove1733449539227,,1733449539263.a1102c9d50dc8624cb9af2bfbf31bc75. 2024-12-06T01:45:41,278 DEBUG [RS_CLOSE_REGION-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1676): Region close journal for a1102c9d50dc8624cb9af2bfbf31bc75: Waiting for close lock at 1733449541267Running coprocessor pre-close hooks at 1733449541267Disabling compacts and flushes for region at 1733449541267Disabling writes for close at 1733449541267Writing region close event to WAL at 1733449541271 (+4 ms)Running coprocessor post-close hooks at 1733449541277 (+6 ms)Closed at 1733449541278 (+1 ms) 2024-12-06T01:45:41,279 INFO [RS_CLOSE_REGION-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegionServer(3302): Adding a1102c9d50dc8624cb9af2bfbf31bc75 move to 00c764f7c3aa,44363,1733449537002 record at close sequenceid=6 2024-12-06T01:45:41,280 DEBUG [bucket-cache-persister {}] bucket.BucketCachePersister(47): Cache is inconsistent, persisting to disk 2024-12-06T01:45:41,281 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1487): Thread bucket-cache-persister started persisting bucket cache to file 2024-12-06T01:45:41,281 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1494): Persist in new chunked persistence format. 2024-12-06T01:45:41,281 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1673): persistToFile: before persisting backing map size: 0, fullycachedFiles size: 0, chunkSize: 10000 2024-12-06T01:45:41,282 INFO [RS_CLOSE_REGION-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] handler.UnassignRegionHandler(157): Closed a1102c9d50dc8624cb9af2bfbf31bc75 2024-12-06T01:45:41,283 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=a1102c9d50dc8624cb9af2bfbf31bc75, regionState=CLOSED 2024-12-06T01:45:41,285 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1680): persistToFile: after persisting backing map size: 0, fullycachedFiles size: 0 2024-12-06T01:45:41,285 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1498): PersistToFile: after persisting backing map size: 0, fullycachedFiles size: 0, file name: bucket.persistence1733449541281 2024-12-06T01:45:41,285 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1509): Thread bucket-cache-persister finished persisting bucket cache to file, renaming 2024-12-06T01:45:41,287 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-4-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=8, ppid=7, state=RUNNABLE, hasLock=false; CloseRegionProcedure a1102c9d50dc8624cb9af2bfbf31bc75, server=00c764f7c3aa,37845,1733449536857 because future has completed 2024-12-06T01:45:41,291 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=8, resume processing ppid=7 2024-12-06T01:45:41,291 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=8, ppid=7, state=SUCCESS, hasLock=false; CloseRegionProcedure a1102c9d50dc8624cb9af2bfbf31bc75, server=00c764f7c3aa,37845,1733449536857 in 182 msec 2024-12-06T01:45:41,291 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1733449539227, region=a1102c9d50dc8624cb9af2bfbf31bc75, REOPEN/MOVE; state=CLOSED, location=00c764f7c3aa,44363,1733449537002; forceNewPlan=false, retain=false 2024-12-06T01:45:41,442 INFO [00c764f7c3aa:40497 {}] balancer.BaseLoadBalancer(388): Reassigned 1 regions. 1 retained the pre-restart assignment. 2024-12-06T01:45:41,442 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=a1102c9d50dc8624cb9af2bfbf31bc75, regionState=OPENING, regionLocation=00c764f7c3aa,44363,1733449537002 2024-12-06T01:45:41,446 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-4-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1733449539227, region=a1102c9d50dc8624cb9af2bfbf31bc75, REOPEN/MOVE because future has completed 2024-12-06T01:45:41,446 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=9, ppid=7, state=RUNNABLE, hasLock=false; OpenRegionProcedure a1102c9d50dc8624cb9af2bfbf31bc75, server=00c764f7c3aa,44363,1733449537002}] 2024-12-06T01:45:41,602 INFO [RS_OPEN_REGION-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] handler.AssignRegionHandler(132): Open testBlockEvictionOnRegionMove1733449539227,,1733449539263.a1102c9d50dc8624cb9af2bfbf31bc75. 2024-12-06T01:45:41,602 DEBUG [RS_OPEN_REGION-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(7752): Opening region: {ENCODED => a1102c9d50dc8624cb9af2bfbf31bc75, NAME => 'testBlockEvictionOnRegionMove1733449539227,,1733449539263.a1102c9d50dc8624cb9af2bfbf31bc75.', STARTKEY => '', ENDKEY => ''} 2024-12-06T01:45:41,603 DEBUG [RS_OPEN_REGION-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testBlockEvictionOnRegionMove1733449539227 a1102c9d50dc8624cb9af2bfbf31bc75 2024-12-06T01:45:41,603 DEBUG [RS_OPEN_REGION-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(898): Instantiated testBlockEvictionOnRegionMove1733449539227,,1733449539263.a1102c9d50dc8624cb9af2bfbf31bc75.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-06T01:45:41,603 DEBUG [RS_OPEN_REGION-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(7794): checking encryption for a1102c9d50dc8624cb9af2bfbf31bc75 2024-12-06T01:45:41,603 DEBUG [RS_OPEN_REGION-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(7797): checking classloading for a1102c9d50dc8624cb9af2bfbf31bc75 2024-12-06T01:45:41,604 INFO [StoreOpener-a1102c9d50dc8624cb9af2bfbf31bc75-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family family of region a1102c9d50dc8624cb9af2bfbf31bc75 2024-12-06T01:45:41,605 INFO [StoreOpener-a1102c9d50dc8624cb9af2bfbf31bc75-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region a1102c9d50dc8624cb9af2bfbf31bc75 columnFamilyName family 2024-12-06T01:45:41,605 DEBUG [StoreOpener-a1102c9d50dc8624cb9af2bfbf31bc75-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T01:45:41,608 DEBUG [StoreFileOpener-a1102c9d50dc8624cb9af2bfbf31bc75-family-1 {}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/eff72b7a-3322-3fdb-88e1-7287cd98d0d4/e93bd390-0caa-6c00-f03a-52237a196617/data/default/testBlockEvictionOnRegionMove1733449539227/a1102c9d50dc8624cb9af2bfbf31bc75/family/6633fe18a90f4711b6189f0d51fa54a8, delay=978 ms 2024-12-06T01:45:41,610 DEBUG [StoreOpener-a1102c9d50dc8624cb9af2bfbf31bc75-1 {}] regionserver.StoreEngine(278): loaded file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/eff72b7a-3322-3fdb-88e1-7287cd98d0d4/e93bd390-0caa-6c00-f03a-52237a196617/data/default/testBlockEvictionOnRegionMove1733449539227/a1102c9d50dc8624cb9af2bfbf31bc75/family/6633fe18a90f4711b6189f0d51fa54a8 2024-12-06T01:45:41,610 INFO [StoreOpener-a1102c9d50dc8624cb9af2bfbf31bc75-1 {}] regionserver.HStore(327): Store=a1102c9d50dc8624cb9af2bfbf31bc75/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-06T01:45:41,610 DEBUG [RS_OPEN_REGION-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1038): replaying wal for a1102c9d50dc8624cb9af2bfbf31bc75 2024-12-06T01:45:41,612 DEBUG [RS_OPEN_REGION-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/eff72b7a-3322-3fdb-88e1-7287cd98d0d4/e93bd390-0caa-6c00-f03a-52237a196617/data/default/testBlockEvictionOnRegionMove1733449539227/a1102c9d50dc8624cb9af2bfbf31bc75 2024-12-06T01:45:41,612 DEBUG [RS_OPEN_REGION-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/eff72b7a-3322-3fdb-88e1-7287cd98d0d4/e93bd390-0caa-6c00-f03a-52237a196617/data/default/testBlockEvictionOnRegionMove1733449539227/a1102c9d50dc8624cb9af2bfbf31bc75 2024-12-06T01:45:41,612 DEBUG [RS_OPEN_REGION-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1048): stopping wal replay for a1102c9d50dc8624cb9af2bfbf31bc75 2024-12-06T01:45:41,613 DEBUG [RS_OPEN_REGION-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1060): Cleaning up temporary data for a1102c9d50dc8624cb9af2bfbf31bc75 2024-12-06T01:45:41,617 DEBUG [RS_OPEN_REGION-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1093): writing seq id for a1102c9d50dc8624cb9af2bfbf31bc75 2024-12-06T01:45:41,618 INFO [RS_OPEN_REGION-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1114): Opened a1102c9d50dc8624cb9af2bfbf31bc75; next sequenceid=10; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=65758525, jitterRate=-0.02012161910533905}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-06T01:45:41,618 DEBUG [RS_OPEN_REGION-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1122): Running coprocessor post-open hooks for a1102c9d50dc8624cb9af2bfbf31bc75 2024-12-06T01:45:41,620 DEBUG [RS_OPEN_REGION-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1006): Region open journal for a1102c9d50dc8624cb9af2bfbf31bc75: Running coprocessor pre-open hook at 1733449541603Writing region info on filesystem at 1733449541603Initializing all the Stores at 1733449541604 (+1 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733449541604Cleaning up temporary data from old regions at 1733449541613 (+9 ms)Running coprocessor post-open hooks at 1733449541618 (+5 ms)Region opened successfully at 1733449541619 (+1 ms) 2024-12-06T01:45:41,620 INFO [RS_OPEN_REGION-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegionServer(2236): Post open deploy tasks for testBlockEvictionOnRegionMove1733449539227,,1733449539263.a1102c9d50dc8624cb9af2bfbf31bc75., pid=9, masterSystemTime=1733449541599 2024-12-06T01:45:41,623 DEBUG [RS_OPEN_REGION-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegionServer(2266): Finished post open deploy task for testBlockEvictionOnRegionMove1733449539227,,1733449539263.a1102c9d50dc8624cb9af2bfbf31bc75. 2024-12-06T01:45:41,623 INFO [RS_OPEN_REGION-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] handler.AssignRegionHandler(153): Opened testBlockEvictionOnRegionMove1733449539227,,1733449539263.a1102c9d50dc8624cb9af2bfbf31bc75. 2024-12-06T01:45:41,624 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=a1102c9d50dc8624cb9af2bfbf31bc75, regionState=OPEN, openSeqNum=10, regionLocation=00c764f7c3aa,44363,1733449537002 2024-12-06T01:45:41,627 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-4-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=9, ppid=7, state=RUNNABLE, hasLock=false; OpenRegionProcedure a1102c9d50dc8624cb9af2bfbf31bc75, server=00c764f7c3aa,44363,1733449537002 because future has completed 2024-12-06T01:45:41,631 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=9, resume processing ppid=7 2024-12-06T01:45:41,631 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=9, ppid=7, state=SUCCESS, hasLock=false; OpenRegionProcedure a1102c9d50dc8624cb9af2bfbf31bc75, server=00c764f7c3aa,44363,1733449537002 in 183 msec 2024-12-06T01:45:41,633 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=7, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1733449539227, region=a1102c9d50dc8624cb9af2bfbf31bc75, REOPEN/MOVE in 532 msec 2024-12-06T01:45:41,653 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster 2024-12-06T01:45:41,654 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-12-06T01:45:41,654 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79) at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611) at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.io.hfile.TestBlockEvictionOnRegionMovement.tearDown(TestBlockEvictionOnRegionMovement.java:174) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-06T01:45:41,659 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-06T01:45:41,659 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-06T01:45:41,660 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-06T01:45:41,660 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2024-12-06T01:45:41,660 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=632300009, stopped=false 2024-12-06T01:45:41,660 INFO [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=00c764f7c3aa,40497,1733449536009 2024-12-06T01:45:41,662 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40497-0x100650089b40000, quorum=127.0.0.1:54114, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-12-06T01:45:41,662 DEBUG [pool-21-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44363-0x100650089b40002, quorum=127.0.0.1:54114, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-12-06T01:45:41,662 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37845-0x100650089b40001, quorum=127.0.0.1:54114, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-12-06T01:45:41,663 DEBUG [pool-21-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44363-0x100650089b40002, quorum=127.0.0.1:54114, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-06T01:45:41,663 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40497-0x100650089b40000, quorum=127.0.0.1:54114, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-06T01:45:41,663 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-06T01:45:41,663 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37845-0x100650089b40001, quorum=127.0.0.1:54114, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-06T01:45:41,664 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-12-06T01:45:41,664 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:37845-0x100650089b40001, quorum=127.0.0.1:54114, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-06T01:45:41,664 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277) at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265) at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416) at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.io.hfile.TestBlockEvictionOnRegionMovement.tearDown(TestBlockEvictionOnRegionMovement.java:174) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-06T01:45:41,664 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:44363-0x100650089b40002, quorum=127.0.0.1:54114, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-06T01:45:41,664 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-06T01:45:41,664 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:40497-0x100650089b40000, quorum=127.0.0.1:54114, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-06T01:45:41,665 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '00c764f7c3aa,37845,1733449536857' ***** 2024-12-06T01:45:41,665 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-12-06T01:45:41,665 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '00c764f7c3aa,44363,1733449537002' ***** 2024-12-06T01:45:41,665 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-12-06T01:45:41,665 INFO [RS:0;00c764f7c3aa:37845 {}] bucket.BucketCache(1738): Disabling cache 2024-12-06T01:45:41,665 INFO [RS:1;00c764f7c3aa:44363 {}] bucket.BucketCache(1738): Disabling cache 2024-12-06T01:45:41,666 INFO [RS:0;00c764f7c3aa:37845 {}] bucket.BucketCache(1762): Shutdown bucket cache: IO persistent=true; path to write=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/eff72b7a-3322-3fdb-88e1-7287cd98d0d4/bucket.persistence 2024-12-06T01:45:41,666 INFO [Time-limited test-BucketCacheWriter-1 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-1 exiting, cacheEnabled=false 2024-12-06T01:45:41,666 INFO [RS:1;00c764f7c3aa:44363 {}] bucket.BucketCache(1762): Shutdown bucket cache: IO persistent=true; path to write=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/eff72b7a-3322-3fdb-88e1-7287cd98d0d4/bucket.persistence 2024-12-06T01:45:41,666 INFO [Time-limited test-BucketCacheWriter-0 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-0 exiting, cacheEnabled=false 2024-12-06T01:45:41,666 INFO [Time-limited test-BucketCacheWriter-0 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-0 exiting, cacheEnabled=false 2024-12-06T01:45:41,666 INFO [Time-limited test-BucketCacheWriter-1 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-1 exiting, cacheEnabled=false 2024-12-06T01:45:41,666 INFO [Time-limited test-BucketCacheWriter-2 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-2 exiting, cacheEnabled=false 2024-12-06T01:45:41,666 INFO [RS:0;00c764f7c3aa:37845 {}] bucket.BucketCache(1768): Shutting down cache persister thread. 2024-12-06T01:45:41,666 INFO [Time-limited test-BucketCacheWriter-2 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-2 exiting, cacheEnabled=false 2024-12-06T01:45:41,666 INFO [RS:1;00c764f7c3aa:44363 {}] bucket.BucketCache(1768): Shutting down cache persister thread. 2024-12-06T01:45:41,666 WARN [bucket-cache-persister {}] bucket.BucketCachePersister(64): Interrupting BucketCachePersister thread. java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCachePersister.run(BucketCachePersister.java:45) ~[classes/:?] 2024-12-06T01:45:41,667 WARN [bucket-cache-persister {}] bucket.BucketCachePersister(64): Interrupting BucketCachePersister thread. java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCachePersister.run(BucketCachePersister.java:45) ~[classes/:?] 2024-12-06T01:45:41,676 DEBUG [RS:0;00c764f7c3aa:37845 {}] bucket.BucketCache(1487): Thread RS:0;00c764f7c3aa:37845 started persisting bucket cache to file 2024-12-06T01:45:41,677 DEBUG [RS:0;00c764f7c3aa:37845 {}] bucket.BucketCache(1494): Persist in new chunked persistence format. 2024-12-06T01:45:41,677 DEBUG [RS:1;00c764f7c3aa:44363 {}] bucket.BucketCache(1487): Thread RS:1;00c764f7c3aa:44363 started persisting bucket cache to file 2024-12-06T01:45:41,677 DEBUG [RS:0;00c764f7c3aa:37845 {}] bucket.BucketCache(1673): persistToFile: before persisting backing map size: 0, fullycachedFiles size: 0, chunkSize: 10000 2024-12-06T01:45:41,677 DEBUG [RS:1;00c764f7c3aa:44363 {}] bucket.BucketCache(1494): Persist in new chunked persistence format. 2024-12-06T01:45:41,677 DEBUG [RS:1;00c764f7c3aa:44363 {}] bucket.BucketCache(1673): persistToFile: before persisting backing map size: 0, fullycachedFiles size: 0, chunkSize: 10000 2024-12-06T01:45:41,681 DEBUG [RS:1;00c764f7c3aa:44363 {}] bucket.BucketCache(1680): persistToFile: after persisting backing map size: 0, fullycachedFiles size: 0 2024-12-06T01:45:41,681 DEBUG [RS:1;00c764f7c3aa:44363 {}] bucket.BucketCache(1498): PersistToFile: after persisting backing map size: 0, fullycachedFiles size: 0, file name: bucket.persistence1733449541677 2024-12-06T01:45:41,681 DEBUG [RS:1;00c764f7c3aa:44363 {}] bucket.BucketCache(1509): Thread RS:1;00c764f7c3aa:44363 finished persisting bucket cache to file, renaming 2024-12-06T01:45:41,681 DEBUG [RS:0;00c764f7c3aa:37845 {}] bucket.BucketCache(1680): persistToFile: after persisting backing map size: 0, fullycachedFiles size: 0 2024-12-06T01:45:41,681 DEBUG [RS:0;00c764f7c3aa:37845 {}] bucket.BucketCache(1498): PersistToFile: after persisting backing map size: 0, fullycachedFiles size: 0, file name: bucket.persistence1733449541676 2024-12-06T01:45:41,681 DEBUG [RS:0;00c764f7c3aa:37845 {}] bucket.BucketCache(1509): Thread RS:0;00c764f7c3aa:37845 finished persisting bucket cache to file, renaming 2024-12-06T01:45:41,681 INFO [RS:1;00c764f7c3aa:44363 {}] regionserver.HeapMemoryManager(220): Stopping 2024-12-06T01:45:41,682 INFO [RS:1;00c764f7c3aa:44363 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-12-06T01:45:41,682 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-12-06T01:45:41,682 INFO [RS:1;00c764f7c3aa:44363 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-12-06T01:45:41,682 INFO [RS:1;00c764f7c3aa:44363 {}] regionserver.HRegionServer(3091): Received CLOSE for a1102c9d50dc8624cb9af2bfbf31bc75 2024-12-06T01:45:41,683 INFO [RS:1;00c764f7c3aa:44363 {}] regionserver.HRegionServer(959): stopping server 00c764f7c3aa,44363,1733449537002 2024-12-06T01:45:41,683 INFO [RS:0;00c764f7c3aa:37845 {}] regionserver.HeapMemoryManager(220): Stopping 2024-12-06T01:45:41,683 INFO [RS:1;00c764f7c3aa:44363 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-12-06T01:45:41,683 INFO [RS:0;00c764f7c3aa:37845 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-12-06T01:45:41,683 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-12-06T01:45:41,683 INFO [RS:1;00c764f7c3aa:44363 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:1;00c764f7c3aa:44363. 2024-12-06T01:45:41,683 INFO [RS:0;00c764f7c3aa:37845 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-12-06T01:45:41,683 INFO [RS:0;00c764f7c3aa:37845 {}] regionserver.HRegionServer(959): stopping server 00c764f7c3aa,37845,1733449536857 2024-12-06T01:45:41,683 DEBUG [RS:1;00c764f7c3aa:44363 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-06T01:45:41,683 INFO [RS:0;00c764f7c3aa:37845 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-12-06T01:45:41,683 DEBUG [RS:1;00c764f7c3aa:44363 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-06T01:45:41,683 INFO [RS:0;00c764f7c3aa:37845 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;00c764f7c3aa:37845. 2024-12-06T01:45:41,683 INFO [RS_CLOSE_REGION-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1662): Closing region testBlockEvictionOnRegionMove1733449539227,,1733449539263.a1102c9d50dc8624cb9af2bfbf31bc75. during a graceful stop, and cache persistence is on, so setting evict on close to false. 2024-12-06T01:45:41,683 DEBUG [RS:0;00c764f7c3aa:37845 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-06T01:45:41,683 DEBUG [RS:0;00c764f7c3aa:37845 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-06T01:45:41,683 INFO [RS:1;00c764f7c3aa:44363 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-12-06T01:45:41,683 INFO [RS:1;00c764f7c3aa:44363 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-12-06T01:45:41,683 INFO [RS:1;00c764f7c3aa:44363 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-12-06T01:45:41,684 DEBUG [RS_CLOSE_REGION-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing a1102c9d50dc8624cb9af2bfbf31bc75, disabling compactions & flushes 2024-12-06T01:45:41,684 INFO [RS:1;00c764f7c3aa:44363 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-12-06T01:45:41,684 INFO [RS:0;00c764f7c3aa:37845 {}] regionserver.HRegionServer(976): stopping server 00c764f7c3aa,37845,1733449536857; all regions closed. 2024-12-06T01:45:41,684 INFO [RS_CLOSE_REGION-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testBlockEvictionOnRegionMove1733449539227,,1733449539263.a1102c9d50dc8624cb9af2bfbf31bc75. 2024-12-06T01:45:41,684 DEBUG [RS_CLOSE_REGION-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testBlockEvictionOnRegionMove1733449539227,,1733449539263.a1102c9d50dc8624cb9af2bfbf31bc75. 2024-12-06T01:45:41,684 DEBUG [RS_CLOSE_REGION-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testBlockEvictionOnRegionMove1733449539227,,1733449539263.a1102c9d50dc8624cb9af2bfbf31bc75. after waiting 0 ms 2024-12-06T01:45:41,684 DEBUG [RS_CLOSE_REGION-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testBlockEvictionOnRegionMove1733449539227,,1733449539263.a1102c9d50dc8624cb9af2bfbf31bc75. 2024-12-06T01:45:41,684 INFO [RS:1;00c764f7c3aa:44363 {}] regionserver.HRegionServer(1321): Waiting on 2 regions to close 2024-12-06T01:45:41,684 INFO [RS_CLOSE_META-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1662): Closing region hbase:meta,,1.1588230740 during a graceful stop, and cache persistence is on, so setting evict on close to false. 2024-12-06T01:45:41,684 DEBUG [RS:1;00c764f7c3aa:44363 {}] regionserver.HRegionServer(1325): Online Regions={1588230740=hbase:meta,,1.1588230740, a1102c9d50dc8624cb9af2bfbf31bc75=testBlockEvictionOnRegionMove1733449539227,,1733449539263.a1102c9d50dc8624cb9af2bfbf31bc75.} 2024-12-06T01:45:41,684 DEBUG [RS_CLOSE_META-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-12-06T01:45:41,684 INFO [RS_CLOSE_META-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-12-06T01:45:41,685 DEBUG [RS_CLOSE_META-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-12-06T01:45:41,685 DEBUG [RS_CLOSE_META-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-12-06T01:45:41,685 DEBUG [RS:1;00c764f7c3aa:44363 {}] regionserver.HRegionServer(1351): Waiting on 1588230740, a1102c9d50dc8624cb9af2bfbf31bc75 2024-12-06T01:45:41,685 DEBUG [RS_CLOSE_META-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-12-06T01:45:41,685 INFO [RS_CLOSE_META-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=3.70 KB heapSize=6.72 KB 2024-12-06T01:45:41,686 DEBUG [StoreFileCloser-a1102c9d50dc8624cb9af2bfbf31bc75-family-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/eff72b7a-3322-3fdb-88e1-7287cd98d0d4/e93bd390-0caa-6c00-f03a-52237a196617/data/default/testBlockEvictionOnRegionMove1733449539227/a1102c9d50dc8624cb9af2bfbf31bc75/family/6633fe18a90f4711b6189f0d51fa54a8 2024-12-06T01:45:41,689 DEBUG [RS:0;00c764f7c3aa:37845 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/eff72b7a-3322-3fdb-88e1-7287cd98d0d4/e93bd390-0caa-6c00-f03a-52237a196617/oldWALs 2024-12-06T01:45:41,689 INFO [RS:0;00c764f7c3aa:37845 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 00c764f7c3aa%2C37845%2C1733449536857:(num 1733449538471) 2024-12-06T01:45:41,689 DEBUG [RS:0;00c764f7c3aa:37845 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-06T01:45:41,689 INFO [RS:0;00c764f7c3aa:37845 {}] regionserver.LeaseManager(133): Closed leases 2024-12-06T01:45:41,690 INFO [RS:0;00c764f7c3aa:37845 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-12-06T01:45:41,690 INFO [RS:0;00c764f7c3aa:37845 {}] hbase.ChoreService(370): Chore service for: regionserver/00c764f7c3aa:0 had [ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-12-06T01:45:41,690 INFO [RS:0;00c764f7c3aa:37845 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-12-06T01:45:41,690 INFO [RS:0;00c764f7c3aa:37845 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-12-06T01:45:41,690 INFO [regionserver/00c764f7c3aa:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-12-06T01:45:41,690 INFO [RS:0;00c764f7c3aa:37845 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-12-06T01:45:41,690 INFO [RS:0;00c764f7c3aa:37845 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-12-06T01:45:41,691 INFO [RS:0;00c764f7c3aa:37845 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:37845 2024-12-06T01:45:41,692 DEBUG [RS_CLOSE_REGION-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/eff72b7a-3322-3fdb-88e1-7287cd98d0d4/e93bd390-0caa-6c00-f03a-52237a196617/data/default/testBlockEvictionOnRegionMove1733449539227/a1102c9d50dc8624cb9af2bfbf31bc75/recovered.edits/12.seqid, newMaxSeqId=12, maxSeqId=9 2024-12-06T01:45:41,693 INFO [RS_CLOSE_REGION-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testBlockEvictionOnRegionMove1733449539227,,1733449539263.a1102c9d50dc8624cb9af2bfbf31bc75. 2024-12-06T01:45:41,693 DEBUG [RS_CLOSE_REGION-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for a1102c9d50dc8624cb9af2bfbf31bc75: Waiting for close lock at 1733449541683Running coprocessor pre-close hooks at 1733449541683Disabling compacts and flushes for region at 1733449541683Disabling writes for close at 1733449541684 (+1 ms)Writing region close event to WAL at 1733449541686 (+2 ms)Running coprocessor post-close hooks at 1733449541693 (+7 ms)Closed at 1733449541693 2024-12-06T01:45:41,693 DEBUG [RS_CLOSE_REGION-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testBlockEvictionOnRegionMove1733449539227,,1733449539263.a1102c9d50dc8624cb9af2bfbf31bc75. 2024-12-06T01:45:41,695 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40497-0x100650089b40000, quorum=127.0.0.1:54114, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-12-06T01:45:41,695 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37845-0x100650089b40001, quorum=127.0.0.1:54114, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/00c764f7c3aa,37845,1733449536857 2024-12-06T01:45:41,695 INFO [RS:0;00c764f7c3aa:37845 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-12-06T01:45:41,697 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [00c764f7c3aa,37845,1733449536857] 2024-12-06T01:45:41,698 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/00c764f7c3aa,37845,1733449536857 already deleted, retry=false 2024-12-06T01:45:41,699 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; 00c764f7c3aa,37845,1733449536857 expired; onlineServers=1 2024-12-06T01:45:41,699 DEBUG [RS_CLOSE_META-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 34d9cc1afaa44d269a5cb68ebc913612 2024-12-06T01:45:41,712 INFO [regionserver/00c764f7c3aa:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-12-06T01:45:41,714 INFO [regionserver/00c764f7c3aa:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-12-06T01:45:41,717 DEBUG [RS_CLOSE_META-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/eff72b7a-3322-3fdb-88e1-7287cd98d0d4/e93bd390-0caa-6c00-f03a-52237a196617/data/hbase/meta/1588230740/.tmp/info/34d9cc1afaa44d269a5cb68ebc913612 is 205, key is testBlockEvictionOnRegionMove1733449539227,,1733449539263.a1102c9d50dc8624cb9af2bfbf31bc75./info:regioninfo/1733449541624/Put/seqid=0 2024-12-06T01:45:41,718 INFO [RS_CLOSE_META-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=3.48 KB at sequenceid=15 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/eff72b7a-3322-3fdb-88e1-7287cd98d0d4/e93bd390-0caa-6c00-f03a-52237a196617/data/hbase/meta/1588230740/.tmp/info/34d9cc1afaa44d269a5cb68ebc913612 2024-12-06T01:45:41,729 DEBUG [RS_CLOSE_META-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 6a98f70196884063af9597dc94e96a2a 2024-12-06T01:45:41,744 DEBUG [RS_CLOSE_META-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/eff72b7a-3322-3fdb-88e1-7287cd98d0d4/e93bd390-0caa-6c00-f03a-52237a196617/data/hbase/meta/1588230740/.tmp/ns/6a98f70196884063af9597dc94e96a2a is 43, key is default/ns:d/1733449538995/Put/seqid=0 2024-12-06T01:45:41,745 INFO [RS_CLOSE_META-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=74 B at sequenceid=15 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/eff72b7a-3322-3fdb-88e1-7287cd98d0d4/e93bd390-0caa-6c00-f03a-52237a196617/data/hbase/meta/1588230740/.tmp/ns/6a98f70196884063af9597dc94e96a2a 2024-12-06T01:45:41,755 DEBUG [RS_CLOSE_META-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFile$WriterFactory(308): Unable to set drop behind on abee9bdea1b840bea4adf873a945e58e 2024-12-06T01:45:41,769 DEBUG [RS_CLOSE_META-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/eff72b7a-3322-3fdb-88e1-7287cd98d0d4/e93bd390-0caa-6c00-f03a-52237a196617/data/hbase/meta/1588230740/.tmp/table/abee9bdea1b840bea4adf873a945e58e is 78, key is testBlockEvictionOnRegionMove1733449539227/table:state/1733449539684/Put/seqid=0 2024-12-06T01:45:41,770 INFO [RS_CLOSE_META-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=148 B at sequenceid=15 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/eff72b7a-3322-3fdb-88e1-7287cd98d0d4/e93bd390-0caa-6c00-f03a-52237a196617/data/hbase/meta/1588230740/.tmp/table/abee9bdea1b840bea4adf873a945e58e 2024-12-06T01:45:41,773 DEBUG [RS_CLOSE_META-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/eff72b7a-3322-3fdb-88e1-7287cd98d0d4/e93bd390-0caa-6c00-f03a-52237a196617/data/hbase/meta/1588230740/.tmp/info/34d9cc1afaa44d269a5cb68ebc913612 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/eff72b7a-3322-3fdb-88e1-7287cd98d0d4/e93bd390-0caa-6c00-f03a-52237a196617/data/hbase/meta/1588230740/info/34d9cc1afaa44d269a5cb68ebc913612 2024-12-06T01:45:41,774 DEBUG [RS_CLOSE_META-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_CLOSE_META}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/eff72b7a-3322-3fdb-88e1-7287cd98d0d4/e93bd390-0caa-6c00-f03a-52237a196617/data/hbase/meta/1588230740/info/34d9cc1afaa44d269a5cb68ebc913612, delay=959 ms 2024-12-06T01:45:41,775 INFO [RS_CLOSE_META-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/eff72b7a-3322-3fdb-88e1-7287cd98d0d4/e93bd390-0caa-6c00-f03a-52237a196617/data/hbase/meta/1588230740/info/34d9cc1afaa44d269a5cb68ebc913612, entries=15, sequenceid=15, filesize=7.6 K 2024-12-06T01:45:41,776 DEBUG [RS_CLOSE_META-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/eff72b7a-3322-3fdb-88e1-7287cd98d0d4/e93bd390-0caa-6c00-f03a-52237a196617/data/hbase/meta/1588230740/.tmp/ns/6a98f70196884063af9597dc94e96a2a as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/eff72b7a-3322-3fdb-88e1-7287cd98d0d4/e93bd390-0caa-6c00-f03a-52237a196617/data/hbase/meta/1588230740/ns/6a98f70196884063af9597dc94e96a2a 2024-12-06T01:45:41,777 DEBUG [RS_CLOSE_META-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_CLOSE_META}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/eff72b7a-3322-3fdb-88e1-7287cd98d0d4/e93bd390-0caa-6c00-f03a-52237a196617/data/hbase/meta/1588230740/ns/6a98f70196884063af9597dc94e96a2a, delay=995 ms 2024-12-06T01:45:41,778 INFO [RS_CLOSE_META-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/eff72b7a-3322-3fdb-88e1-7287cd98d0d4/e93bd390-0caa-6c00-f03a-52237a196617/data/hbase/meta/1588230740/ns/6a98f70196884063af9597dc94e96a2a, entries=2, sequenceid=15, filesize=5.0 K 2024-12-06T01:45:41,779 DEBUG [RS_CLOSE_META-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/eff72b7a-3322-3fdb-88e1-7287cd98d0d4/e93bd390-0caa-6c00-f03a-52237a196617/data/hbase/meta/1588230740/.tmp/table/abee9bdea1b840bea4adf873a945e58e as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/eff72b7a-3322-3fdb-88e1-7287cd98d0d4/e93bd390-0caa-6c00-f03a-52237a196617/data/hbase/meta/1588230740/table/abee9bdea1b840bea4adf873a945e58e 2024-12-06T01:45:41,780 DEBUG [RS_CLOSE_META-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_CLOSE_META}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/eff72b7a-3322-3fdb-88e1-7287cd98d0d4/e93bd390-0caa-6c00-f03a-52237a196617/data/hbase/meta/1588230740/table/abee9bdea1b840bea4adf873a945e58e, delay=974 ms 2024-12-06T01:45:41,781 INFO [RS_CLOSE_META-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/eff72b7a-3322-3fdb-88e1-7287cd98d0d4/e93bd390-0caa-6c00-f03a-52237a196617/data/hbase/meta/1588230740/table/abee9bdea1b840bea4adf873a945e58e, entries=2, sequenceid=15, filesize=5.3 K 2024-12-06T01:45:41,782 INFO [RS_CLOSE_META-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~3.70 KB/3789, heapSize ~6.42 KB/6576, currentSize=0 B/0 for 1588230740 in 97ms, sequenceid=15, compaction requested=false 2024-12-06T01:45:41,783 DEBUG [RS_CLOSE_META-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_CLOSE_META}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta' 2024-12-06T01:45:41,784 DEBUG [StoreFileCloser-1588230740-info-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/eff72b7a-3322-3fdb-88e1-7287cd98d0d4/e93bd390-0caa-6c00-f03a-52237a196617/data/hbase/meta/1588230740/info/34d9cc1afaa44d269a5cb68ebc913612 2024-12-06T01:45:41,785 DEBUG [StoreFileCloser-1588230740-ns-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/eff72b7a-3322-3fdb-88e1-7287cd98d0d4/e93bd390-0caa-6c00-f03a-52237a196617/data/hbase/meta/1588230740/ns/6a98f70196884063af9597dc94e96a2a 2024-12-06T01:45:41,786 DEBUG [StoreFileCloser-1588230740-table-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/eff72b7a-3322-3fdb-88e1-7287cd98d0d4/e93bd390-0caa-6c00-f03a-52237a196617/data/hbase/meta/1588230740/table/abee9bdea1b840bea4adf873a945e58e 2024-12-06T01:45:41,791 DEBUG [RS_CLOSE_META-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/eff72b7a-3322-3fdb-88e1-7287cd98d0d4/e93bd390-0caa-6c00-f03a-52237a196617/data/hbase/meta/1588230740/recovered.edits/18.seqid, newMaxSeqId=18, maxSeqId=1 2024-12-06T01:45:41,791 DEBUG [RS_CLOSE_META-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-12-06T01:45:41,792 INFO [RS_CLOSE_META-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-12-06T01:45:41,792 DEBUG [RS_CLOSE_META-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1733449541684Running coprocessor pre-close hooks at 1733449541684Disabling compacts and flushes for region at 1733449541684Disabling writes for close at 1733449541685 (+1 ms)Obtaining lock to block concurrent updates at 1733449541685Preparing flush snapshotting stores in 1588230740 at 1733449541685Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=3789, getHeapSize=6816, getOffHeapSize=0, getCellsCount=27 at 1733449541686 (+1 ms)Flushing stores of hbase:meta,,1.1588230740 at 1733449541686Flushing 1588230740/info: creating writer at 1733449541686Flushing 1588230740/info: appending metadata at 1733449541716 (+30 ms)Flushing 1588230740/info: closing flushed file at 1733449541716Flushing 1588230740/ns: creating writer at 1733449541722 (+6 ms)Flushing 1588230740/ns: appending metadata at 1733449541743 (+21 ms)Flushing 1588230740/ns: closing flushed file at 1733449541743Flushing 1588230740/table: creating writer at 1733449541748 (+5 ms)Flushing 1588230740/table: appending metadata at 1733449541769 (+21 ms)Flushing 1588230740/table: closing flushed file at 1733449541769Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@bbf03d7: reopening flushed file at 1733449541772 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@7f8e7e64: reopening flushed file at 1733449541776 (+4 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@7ba3c078: reopening flushed file at 1733449541779 (+3 ms)Finished flush of dataSize ~3.70 KB/3789, heapSize ~6.42 KB/6576, currentSize=0 B/0 for 1588230740 in 97ms, sequenceid=15, compaction requested=false at 1733449541783 (+4 ms)Writing region close event to WAL at 1733449541786 (+3 ms)Running coprocessor post-close hooks at 1733449541791 (+5 ms)Closed at 1733449541792 (+1 ms) 2024-12-06T01:45:41,792 DEBUG [RS_CLOSE_META-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-12-06T01:45:41,798 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37845-0x100650089b40001, quorum=127.0.0.1:54114, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-06T01:45:41,798 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37845-0x100650089b40001, quorum=127.0.0.1:54114, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-06T01:45:41,799 INFO [RS:0;00c764f7c3aa:37845 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-12-06T01:45:41,799 INFO [RS:0;00c764f7c3aa:37845 {}] regionserver.HRegionServer(1031): Exiting; stopping=00c764f7c3aa,37845,1733449536857; zookeeper connection closed. 2024-12-06T01:45:41,799 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@5bd4e0db {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@5bd4e0db 2024-12-06T01:45:41,885 INFO [RS:1;00c764f7c3aa:44363 {}] regionserver.HRegionServer(976): stopping server 00c764f7c3aa,44363,1733449537002; all regions closed. 2024-12-06T01:45:41,888 DEBUG [RS:1;00c764f7c3aa:44363 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/eff72b7a-3322-3fdb-88e1-7287cd98d0d4/e93bd390-0caa-6c00-f03a-52237a196617/oldWALs 2024-12-06T01:45:41,888 INFO [RS:1;00c764f7c3aa:44363 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 00c764f7c3aa%2C44363%2C1733449537002.meta:.meta(num 1733449538854) 2024-12-06T01:45:41,890 DEBUG [RS:1;00c764f7c3aa:44363 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/eff72b7a-3322-3fdb-88e1-7287cd98d0d4/e93bd390-0caa-6c00-f03a-52237a196617/oldWALs 2024-12-06T01:45:41,890 INFO [RS:1;00c764f7c3aa:44363 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 00c764f7c3aa%2C44363%2C1733449537002:(num 1733449538471) 2024-12-06T01:45:41,890 DEBUG [RS:1;00c764f7c3aa:44363 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-06T01:45:41,890 INFO [RS:1;00c764f7c3aa:44363 {}] regionserver.LeaseManager(133): Closed leases 2024-12-06T01:45:41,890 INFO [RS:1;00c764f7c3aa:44363 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-12-06T01:45:41,890 INFO [RS:1;00c764f7c3aa:44363 {}] hbase.ChoreService(370): Chore service for: regionserver/00c764f7c3aa:0 had [ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-12-06T01:45:41,890 INFO [RS:1;00c764f7c3aa:44363 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-12-06T01:45:41,890 INFO [regionserver/00c764f7c3aa:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-12-06T01:45:41,890 INFO [RS:1;00c764f7c3aa:44363 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:44363 2024-12-06T01:45:41,893 DEBUG [pool-21-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44363-0x100650089b40002, quorum=127.0.0.1:54114, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/00c764f7c3aa,44363,1733449537002 2024-12-06T01:45:41,893 INFO [RS:1;00c764f7c3aa:44363 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-12-06T01:45:41,894 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40497-0x100650089b40000, quorum=127.0.0.1:54114, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-12-06T01:45:41,895 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [00c764f7c3aa,44363,1733449537002] 2024-12-06T01:45:41,896 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/00c764f7c3aa,44363,1733449537002 already deleted, retry=false 2024-12-06T01:45:41,896 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; 00c764f7c3aa,44363,1733449537002 expired; onlineServers=0 2024-12-06T01:45:41,897 INFO [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master '00c764f7c3aa,40497,1733449536009' ***** 2024-12-06T01:45:41,897 INFO [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0 2024-12-06T01:45:41,897 INFO [M:0;00c764f7c3aa:40497 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-12-06T01:45:41,897 INFO [M:0;00c764f7c3aa:40497 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-12-06T01:45:41,897 DEBUG [M:0;00c764f7c3aa:40497 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2024-12-06T01:45:41,897 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2024-12-06T01:45:41,897 DEBUG [M:0;00c764f7c3aa:40497 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2024-12-06T01:45:41,897 DEBUG [master/00c764f7c3aa:0:becomeActiveMaster-HFileCleaner.small.0-1733449538113 {}] cleaner.HFileCleaner(306): Exit Thread[master/00c764f7c3aa:0:becomeActiveMaster-HFileCleaner.small.0-1733449538113,5,FailOnTimeoutGroup] 2024-12-06T01:45:41,897 DEBUG [master/00c764f7c3aa:0:becomeActiveMaster-HFileCleaner.large.0-1733449538112 {}] cleaner.HFileCleaner(306): Exit Thread[master/00c764f7c3aa:0:becomeActiveMaster-HFileCleaner.large.0-1733449538112,5,FailOnTimeoutGroup] 2024-12-06T01:45:41,898 INFO [M:0;00c764f7c3aa:40497 {}] hbase.ChoreService(370): Chore service for: master/00c764f7c3aa:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown 2024-12-06T01:45:41,898 INFO [M:0;00c764f7c3aa:40497 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-12-06T01:45:41,898 DEBUG [M:0;00c764f7c3aa:40497 {}] master.HMaster(1795): Stopping service threads 2024-12-06T01:45:41,898 INFO [M:0;00c764f7c3aa:40497 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2024-12-06T01:45:41,898 INFO [M:0;00c764f7c3aa:40497 {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-06T01:45:41,899 INFO [M:0;00c764f7c3aa:40497 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-12-06T01:45:41,899 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2024-12-06T01:45:41,900 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40497-0x100650089b40000, quorum=127.0.0.1:54114, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-12-06T01:45:41,900 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40497-0x100650089b40000, quorum=127.0.0.1:54114, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-06T01:45:41,900 DEBUG [M:0;00c764f7c3aa:40497 {}] zookeeper.ZKUtil(347): master:40497-0x100650089b40000, quorum=127.0.0.1:54114, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error) 2024-12-06T01:45:41,900 WARN [M:0;00c764f7c3aa:40497 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null 2024-12-06T01:45:41,901 INFO [M:0;00c764f7c3aa:40497 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/eff72b7a-3322-3fdb-88e1-7287cd98d0d4/e93bd390-0caa-6c00-f03a-52237a196617/.lastflushedseqids 2024-12-06T01:45:41,914 INFO [M:0;00c764f7c3aa:40497 {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-12-06T01:45:41,914 INFO [M:0;00c764f7c3aa:40497 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2024-12-06T01:45:41,915 DEBUG [M:0;00c764f7c3aa:40497 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-12-06T01:45:41,915 INFO [M:0;00c764f7c3aa:40497 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-06T01:45:41,915 DEBUG [M:0;00c764f7c3aa:40497 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-06T01:45:41,915 DEBUG [M:0;00c764f7c3aa:40497 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-12-06T01:45:41,915 DEBUG [M:0;00c764f7c3aa:40497 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-06T01:45:41,915 INFO [M:0;00c764f7c3aa:40497 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=34.66 KB heapSize=42.99 KB 2024-12-06T01:45:41,926 DEBUG [M:0;00c764f7c3aa:40497 {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on aeb1564163ef46008991aa5aac4ee672 2024-12-06T01:45:41,944 DEBUG [M:0;00c764f7c3aa:40497 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/eff72b7a-3322-3fdb-88e1-7287cd98d0d4/e93bd390-0caa-6c00-f03a-52237a196617/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/aeb1564163ef46008991aa5aac4ee672 is 82, key is hbase:meta,,1/info:regioninfo/1733449538929/Put/seqid=0 2024-12-06T01:45:41,945 INFO [M:0;00c764f7c3aa:40497 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=504 B at sequenceid=84 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/eff72b7a-3322-3fdb-88e1-7287cd98d0d4/e93bd390-0caa-6c00-f03a-52237a196617/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/aeb1564163ef46008991aa5aac4ee672 2024-12-06T01:45:41,953 DEBUG [M:0;00c764f7c3aa:40497 {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 09f5bfe5465744da86adb57d26e0d9c9 2024-12-06T01:45:41,968 DEBUG [M:0;00c764f7c3aa:40497 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/eff72b7a-3322-3fdb-88e1-7287cd98d0d4/e93bd390-0caa-6c00-f03a-52237a196617/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/09f5bfe5465744da86adb57d26e0d9c9 is 777, key is \x00\x00\x00\x00\x00\x00\x00\x04/proc:d/1733449539689/Put/seqid=0 2024-12-06T01:45:41,969 INFO [M:0;00c764f7c3aa:40497 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=34.04 KB at sequenceid=84 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/eff72b7a-3322-3fdb-88e1-7287cd98d0d4/e93bd390-0caa-6c00-f03a-52237a196617/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/09f5bfe5465744da86adb57d26e0d9c9 2024-12-06T01:45:41,977 DEBUG [M:0;00c764f7c3aa:40497 {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 8d28fa683c164578b5ac2bae051aff7a 2024-12-06T01:45:41,991 DEBUG [M:0;00c764f7c3aa:40497 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/eff72b7a-3322-3fdb-88e1-7287cd98d0d4/e93bd390-0caa-6c00-f03a-52237a196617/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/8d28fa683c164578b5ac2bae051aff7a is 69, key is 00c764f7c3aa,37845,1733449536857/rs:state/1733449538204/Put/seqid=0 2024-12-06T01:45:41,992 INFO [M:0;00c764f7c3aa:40497 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=130 B at sequenceid=84 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/eff72b7a-3322-3fdb-88e1-7287cd98d0d4/e93bd390-0caa-6c00-f03a-52237a196617/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/8d28fa683c164578b5ac2bae051aff7a 2024-12-06T01:45:41,994 DEBUG [M:0;00c764f7c3aa:40497 {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/eff72b7a-3322-3fdb-88e1-7287cd98d0d4/e93bd390-0caa-6c00-f03a-52237a196617/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/aeb1564163ef46008991aa5aac4ee672 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/eff72b7a-3322-3fdb-88e1-7287cd98d0d4/e93bd390-0caa-6c00-f03a-52237a196617/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/aeb1564163ef46008991aa5aac4ee672 2024-12-06T01:45:41,995 DEBUG [pool-21-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44363-0x100650089b40002, quorum=127.0.0.1:54114, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-06T01:45:41,995 INFO [RS:1;00c764f7c3aa:44363 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-12-06T01:45:41,995 DEBUG [pool-21-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44363-0x100650089b40002, quorum=127.0.0.1:54114, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-06T01:45:41,995 INFO [RS:1;00c764f7c3aa:44363 {}] regionserver.HRegionServer(1031): Exiting; stopping=00c764f7c3aa,44363,1733449537002; zookeeper connection closed. 2024-12-06T01:45:41,995 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@7a74c1e7 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@7a74c1e7 2024-12-06T01:45:41,996 INFO [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 2 regionserver(s) complete 2024-12-06T01:45:41,997 INFO [M:0;00c764f7c3aa:40497 {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/eff72b7a-3322-3fdb-88e1-7287cd98d0d4/e93bd390-0caa-6c00-f03a-52237a196617/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/aeb1564163ef46008991aa5aac4ee672, entries=8, sequenceid=84, filesize=5.5 K 2024-12-06T01:45:41,998 DEBUG [M:0;00c764f7c3aa:40497 {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/eff72b7a-3322-3fdb-88e1-7287cd98d0d4/e93bd390-0caa-6c00-f03a-52237a196617/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/09f5bfe5465744da86adb57d26e0d9c9 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/eff72b7a-3322-3fdb-88e1-7287cd98d0d4/e93bd390-0caa-6c00-f03a-52237a196617/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/09f5bfe5465744da86adb57d26e0d9c9 2024-12-06T01:45:41,999 INFO [M:0;00c764f7c3aa:40497 {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/eff72b7a-3322-3fdb-88e1-7287cd98d0d4/e93bd390-0caa-6c00-f03a-52237a196617/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/09f5bfe5465744da86adb57d26e0d9c9, entries=9, sequenceid=84, filesize=6.5 K 2024-12-06T01:45:42,000 DEBUG [M:0;00c764f7c3aa:40497 {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/eff72b7a-3322-3fdb-88e1-7287cd98d0d4/e93bd390-0caa-6c00-f03a-52237a196617/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/8d28fa683c164578b5ac2bae051aff7a as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/eff72b7a-3322-3fdb-88e1-7287cd98d0d4/e93bd390-0caa-6c00-f03a-52237a196617/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/8d28fa683c164578b5ac2bae051aff7a 2024-12-06T01:45:42,002 INFO [M:0;00c764f7c3aa:40497 {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/eff72b7a-3322-3fdb-88e1-7287cd98d0d4/e93bd390-0caa-6c00-f03a-52237a196617/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/8d28fa683c164578b5ac2bae051aff7a, entries=2, sequenceid=84, filesize=5.1 K 2024-12-06T01:45:42,003 INFO [M:0;00c764f7c3aa:40497 {}] regionserver.HRegion(3140): Finished flush of dataSize ~34.66 KB/35494, heapSize ~42.70 KB/43720, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 88ms, sequenceid=84, compaction requested=false 2024-12-06T01:45:42,008 INFO [M:0;00c764f7c3aa:40497 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-06T01:45:42,008 DEBUG [M:0;00c764f7c3aa:40497 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1733449541915Disabling compacts and flushes for region at 1733449541915Disabling writes for close at 1733449541915Obtaining lock to block concurrent updates at 1733449541915Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1733449541915Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=35494, getHeapSize=43960, getOffHeapSize=0, getCellsCount=99 at 1733449541916 (+1 ms)Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1733449541916Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1733449541916Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1733449541943 (+27 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1733449541944 (+1 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1733449541947 (+3 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1733449541967 (+20 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1733449541967Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1733449541971 (+4 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1733449541991 (+20 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1733449541991Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@21c707a0: reopening flushed file at 1733449541994 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@30a4efa7: reopening flushed file at 1733449541997 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@11fd01e9: reopening flushed file at 1733449542000 (+3 ms)Finished flush of dataSize ~34.66 KB/35494, heapSize ~42.70 KB/43720, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 88ms, sequenceid=84, compaction requested=false at 1733449542003 (+3 ms)Writing region close event to WAL at 1733449542007 (+4 ms)Closed at 1733449542007 2024-12-06T01:45:42,010 INFO [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-12-06T01:45:42,010 INFO [M:0;00c764f7c3aa:40497 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down. 2024-12-06T01:45:42,010 INFO [M:0;00c764f7c3aa:40497 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:40497 2024-12-06T01:45:42,010 INFO [M:0;00c764f7c3aa:40497 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-12-06T01:45:42,113 INFO [M:0;00c764f7c3aa:40497 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-12-06T01:45:42,113 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40497-0x100650089b40000, quorum=127.0.0.1:54114, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-06T01:45:42,113 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40497-0x100650089b40000, quorum=127.0.0.1:54114, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-06T01:45:42,117 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers 2024-12-06T01:45:42,137 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down 2024-12-06T01:45:42,138 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-12-06T01:45:42,146 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: io.hfile.TestBlockEvictionOnRegionMovement#testBlockEvictionOnRegionMove Thread=52 (was 12) Potentially hanging thread: HMaster-EventLoopGroup-1-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-3-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RpcClient-timer-pool-0 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.waitForNextTick(HashedWheelTimer.java:598) app//org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:494) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-4-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: regionserver/00c764f7c3aa:0.procedureResultReporter java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.RemoteProcedureResultReporter.run(RemoteProcedureResultReporter.java:75) Potentially hanging thread: org.apache.hadoop.fs.FileSystem$Statistics$StatisticsDataReferenceCleaner java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:155) java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:176) app//org.apache.hadoop.fs.FileSystem$Statistics$StatisticsDataReferenceCleaner.run(FileSystem.java:4171) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-4-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: SnapshotHandlerChoreCleaner java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: hfile-prefetch-1733449541608 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1177) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: master/00c764f7c3aa:0:becomeActiveMaster-MemStoreChunkPool Statistics java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ForkJoinPool.commonPool-worker-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.ForkJoinPool.awaitWork(ForkJoinPool.java:1724) java.base@17.0.11/java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1623) java.base@17.0.11/java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:165) Potentially hanging thread: region-location-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1170) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-3-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Idle-Rpc-Conn-Sweeper-pool-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HBase-Metrics2-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ForkJoinPool.commonPool-worker-3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.ForkJoinPool.awaitWork(ForkJoinPool.java:1724) java.base@17.0.11/java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1623) java.base@17.0.11/java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:165) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-4 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-5 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: hfile-prefetch-1733449541774 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1177) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: master/00c764f7c3aa:0:becomeActiveMaster-MemStoreChunkPool Statistics java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HMaster-EventLoopGroup-1-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Async-Client-Retry-Timer-pool-0 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.waitForNextTick(HashedWheelTimer.java:598) app//org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:494) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HMaster-EventLoopGroup-1-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-3-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Time-limited test.named-queue-events-pool-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) app//com.lmax.disruptor.BlockingWaitStrategy.waitFor(BlockingWaitStrategy.java:47) app//com.lmax.disruptor.ProcessingSequenceBarrier.waitFor(ProcessingSequenceBarrier.java:56) app//com.lmax.disruptor.BatchEventProcessor.processEvents(BatchEventProcessor.java:159) app//com.lmax.disruptor.BatchEventProcessor.run(BatchEventProcessor.java:125) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: region-location-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1170) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-4-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ForkJoinPool.commonPool-worker-4 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkUntil(LockSupport.java:410) java.base@17.0.11/java.util.concurrent.ForkJoinPool.awaitWork(ForkJoinPool.java:1726) java.base@17.0.11/java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1623) java.base@17.0.11/java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:165) Potentially hanging thread: Monitor thread for TaskMonitor java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hbase.monitoring.TaskMonitor$MonitorRunnable.run(TaskMonitor.java:325) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: hfile-prefetch-1733449541777 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1177) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: regionserver/00c764f7c3aa:0.procedureResultReporter java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.RemoteProcedureResultReporter.run(RemoteProcedureResultReporter.java:75) Potentially hanging thread: weak-ref-cleaner-strictcontextstorage java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:155) java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:176) app//io.opentelemetry.context.StrictContextStorage$PendingScopes.run(StrictContextStorage.java:269) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: hfile-prefetch-1733449540075 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ForkJoinPool.commonPool-worker-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.ForkJoinPool.awaitWork(ForkJoinPool.java:1724) java.base@17.0.11/java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1623) java.base@17.0.11/java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:165) Potentially hanging thread: SessionTracker java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.zookeeper.server.SessionTrackerImpl.run(SessionTrackerImpl.java:163) Potentially hanging thread: Timer for 'HBase' metrics system java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.util.TimerThread.mainLoop(Timer.java:563) java.base@17.0.11/java.util.TimerThread.run(Timer.java:516) - Thread LEAK? -, OpenFileDescriptor=346 (was 287) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=527 (was 578), ProcessCount=11 (was 11), AvailableMemoryMB=10113 (was 10459) 2024-12-06T01:45:42,153 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: io.hfile.TestBlockEvictionOnRegionMovement#testBlockEvictionOnGracefulStop Thread=52, OpenFileDescriptor=346, MaxFileDescriptor=1048576, SystemLoadAverage=527, ProcessCount=11, AvailableMemoryMB=10113 2024-12-06T01:45:42,153 INFO [Time-limited test {}] hbase.HBaseTestingUtil(401): System.getProperty("hadoop.log.dir") already set to: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/eff72b7a-3322-3fdb-88e1-7287cd98d0d4/hadoop-log-dir so I do NOT create it in target/test-data/21f224f3-974a-889b-42d4-cdd5a08d3a54 2024-12-06T01:45:42,153 INFO [Time-limited test {}] hbase.HBaseTestingUtil(401): System.getProperty("hadoop.tmp.dir") already set to: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/eff72b7a-3322-3fdb-88e1-7287cd98d0d4/hadoop-tmp-dir so I do NOT create it in target/test-data/21f224f3-974a-889b-42d4-cdd5a08d3a54 2024-12-06T01:45:42,153 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-12-06T01:45:42,156 INFO [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/21f224f3-974a-889b-42d4-cdd5a08d3a54/cluster_4eddf911-5f65-3233-2188-328cb7d99453, deleteOnExit=true 2024-12-06T01:45:42,160 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/21f224f3-974a-889b-42d4-cdd5a08d3a54/cluster_4eddf911-5f65-3233-2188-328cb7d99453/zookeeper_0, clientPort=61049, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/21f224f3-974a-889b-42d4-cdd5a08d3a54/cluster_4eddf911-5f65-3233-2188-328cb7d99453/zookeeper_0/version-2, dataDirSize=457 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/21f224f3-974a-889b-42d4-cdd5a08d3a54/cluster_4eddf911-5f65-3233-2188-328cb7d99453/zookeeper_0/version-2, dataLogSize=457 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0 2024-12-06T01:45:42,161 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=61049 2024-12-06T01:45:42,173 INFO [Time-limited test {}] util.FSUtils(489): Created version file at /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/eff72b7a-3322-3fdb-88e1-7287cd98d0d4/e93bd390-0caa-6c00-f03a-52237a196617 with version=8 2024-12-06T01:45:42,174 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1141): The hbase.fs.tmp.dir is set to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/eff72b7a-3322-3fdb-88e1-7287cd98d0d4/e93bd390-0caa-6c00-f03a-52237a196617/hbase-staging 2024-12-06T01:45:42,177 INFO [Time-limited test {}] client.ConnectionUtils(128): master/00c764f7c3aa:0 server-side Connection retries=45 2024-12-06T01:45:42,177 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-06T01:45:42,178 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-12-06T01:45:42,178 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-12-06T01:45:42,178 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-06T01:45:42,178 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-12-06T01:45:42,178 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService 2024-12-06T01:45:42,178 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-12-06T01:45:42,179 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:37657 2024-12-06T01:45:42,181 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:37657 connecting to ZooKeeper ensemble=127.0.0.1:61049 2024-12-06T01:45:42,187 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:376570x0, quorum=127.0.0.1:61049, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-06T01:45:42,188 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:37657-0x1006500a2f60000 connected 2024-12-06T01:45:42,203 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-12-06T01:45:42,203 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-12-06T01:45:42,205 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:37657-0x1006500a2f60000, quorum=127.0.0.1:61049, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-06T01:45:42,205 INFO [Time-limited test {}] master.HMaster(525): hbase.rootdir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/eff72b7a-3322-3fdb-88e1-7287cd98d0d4/e93bd390-0caa-6c00-f03a-52237a196617, hbase.cluster.distributed=false 2024-12-06T01:45:42,207 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:37657-0x1006500a2f60000, quorum=127.0.0.1:61049, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-12-06T01:45:42,207 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=37657 2024-12-06T01:45:42,208 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=37657 2024-12-06T01:45:42,208 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=37657 2024-12-06T01:45:42,208 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=37657 2024-12-06T01:45:42,208 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=37657 2024-12-06T01:45:42,232 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/00c764f7c3aa:0 server-side Connection retries=45 2024-12-06T01:45:42,232 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-06T01:45:42,232 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-12-06T01:45:42,232 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-12-06T01:45:42,232 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-06T01:45:42,232 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-12-06T01:45:42,232 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-12-06T01:45:42,233 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-12-06T01:45:42,233 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:46757 2024-12-06T01:45:42,235 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:46757 connecting to ZooKeeper ensemble=127.0.0.1:61049 2024-12-06T01:45:42,235 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-12-06T01:45:42,235 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-12-06T01:45:42,239 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:467570x0, quorum=127.0.0.1:61049, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-06T01:45:42,240 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:46757-0x1006500a2f60001 connected 2024-12-06T01:45:42,240 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:46757-0x1006500a2f60001, quorum=127.0.0.1:61049, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-06T01:45:42,241 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-12-06T01:45:42,241 INFO [Time-limited test {}] bucket.FileIOEngine(99): Allocating cache 400 MB, on the path:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/21f224f3-974a-889b-42d4-cdd5a08d3a54/bucket.cache 2024-12-06T01:45:42,241 INFO [Time-limited test {}] bucket.BucketCache(364): Instantiating BucketCache with acceptableFactor: 0.95, minFactor: 0.85, extraFreeFactor: 0.1, singleFactor: 0.25, multiFactor: 0.5, memoryFactor: 0.25, useStrongRef: false 2024-12-06T01:45:42,242 INFO [Time-limited test {}] bucket.BucketCache(478): Starting BucketCachePersisterThread 2024-12-06T01:45:42,242 INFO [Time-limited test {}] bucket.BucketCachePersister(38): BucketCachePersister started with interval: 100 2024-12-06T01:45:42,242 INFO [Thread-119 {}] bucket.BucketCache(1530): Started retrieving bucket cache from file 2024-12-06T01:45:42,242 WARN [Thread-119 {}] bucket.BucketCache(1533): Persistence file missing! It's ok if it's first run after enabling persistent cache. 2024-12-06T01:45:42,243 INFO [Thread-119 {}] bucket.BucketAllocator(343): Cache totalSize=418148352, buckets=199, bucket capacity=2101248=(4*525312)=(FEWEST_ITEMS_IN_BUCKET*(largest configured bucketcache size)) 2024-12-06T01:45:42,243 INFO [Thread-119 {}] bucket.BucketCache(413): Persistent bucket cache recovery from /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/21f224f3-974a-889b-42d4-cdd5a08d3a54/bucket.persistence is complete. 2024-12-06T01:45:42,245 INFO [Time-limited test {}] bucket.BucketCache(403): Started bucket cache; ioengine=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/21f224f3-974a-889b-42d4-cdd5a08d3a54/bucket.cache, capacity=400 MB, blockSize=64 KB, writerThreadNum=3, writerQLen=64, persistencePath=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/21f224f3-974a-889b-42d4-cdd5a08d3a54/bucket.persistence, bucketAllocator=org.apache.hadoop.hbase.io.hfile.bucket.BucketAllocator 2024-12-06T01:45:42,245 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-12-06T01:45:42,246 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:46757-0x1006500a2f60001, quorum=127.0.0.1:61049, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-12-06T01:45:42,247 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:46757-0x1006500a2f60001, quorum=127.0.0.1:61049, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-12-06T01:45:42,248 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=46757 2024-12-06T01:45:42,248 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=46757 2024-12-06T01:45:42,248 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=46757 2024-12-06T01:45:42,249 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=46757 2024-12-06T01:45:42,249 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=46757 2024-12-06T01:45:42,265 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/00c764f7c3aa:0 server-side Connection retries=45 2024-12-06T01:45:42,265 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-06T01:45:42,265 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-12-06T01:45:42,265 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-12-06T01:45:42,265 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-06T01:45:42,266 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-12-06T01:45:42,266 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-12-06T01:45:42,266 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-12-06T01:45:42,267 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:45953 2024-12-06T01:45:42,268 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:45953 connecting to ZooKeeper ensemble=127.0.0.1:61049 2024-12-06T01:45:42,268 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-12-06T01:45:42,268 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-12-06T01:45:42,272 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:459530x0, quorum=127.0.0.1:61049, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-06T01:45:42,272 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:45953-0x1006500a2f60002 connected 2024-12-06T01:45:42,272 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:45953-0x1006500a2f60002, quorum=127.0.0.1:61049, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-06T01:45:42,273 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-12-06T01:45:42,273 INFO [Time-limited test {}] bucket.FileIOEngine(99): Allocating cache 400 MB, on the path:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/21f224f3-974a-889b-42d4-cdd5a08d3a54/bucket.cache 2024-12-06T01:45:42,273 INFO [Time-limited test {}] bucket.BucketCache(364): Instantiating BucketCache with acceptableFactor: 0.95, minFactor: 0.85, extraFreeFactor: 0.1, singleFactor: 0.25, multiFactor: 0.5, memoryFactor: 0.25, useStrongRef: false 2024-12-06T01:45:42,274 INFO [Time-limited test {}] bucket.BucketCache(478): Starting BucketCachePersisterThread 2024-12-06T01:45:42,274 INFO [Time-limited test {}] bucket.BucketCachePersister(38): BucketCachePersister started with interval: 100 2024-12-06T01:45:42,274 INFO [Thread-120 {}] bucket.BucketCache(1530): Started retrieving bucket cache from file 2024-12-06T01:45:42,274 WARN [Thread-120 {}] bucket.BucketCache(1533): Persistence file missing! It's ok if it's first run after enabling persistent cache. 2024-12-06T01:45:42,274 INFO [Thread-120 {}] bucket.BucketAllocator(343): Cache totalSize=418148352, buckets=199, bucket capacity=2101248=(4*525312)=(FEWEST_ITEMS_IN_BUCKET*(largest configured bucketcache size)) 2024-12-06T01:45:42,274 INFO [Thread-120 {}] bucket.BucketCache(413): Persistent bucket cache recovery from /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/21f224f3-974a-889b-42d4-cdd5a08d3a54/bucket.persistence is complete. 2024-12-06T01:45:42,276 INFO [Time-limited test {}] bucket.BucketCache(403): Started bucket cache; ioengine=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/21f224f3-974a-889b-42d4-cdd5a08d3a54/bucket.cache, capacity=400 MB, blockSize=64 KB, writerThreadNum=3, writerQLen=64, persistencePath=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/21f224f3-974a-889b-42d4-cdd5a08d3a54/bucket.persistence, bucketAllocator=org.apache.hadoop.hbase.io.hfile.bucket.BucketAllocator 2024-12-06T01:45:42,276 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-12-06T01:45:42,277 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:45953-0x1006500a2f60002, quorum=127.0.0.1:61049, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-12-06T01:45:42,278 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:45953-0x1006500a2f60002, quorum=127.0.0.1:61049, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-12-06T01:45:42,279 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=45953 2024-12-06T01:45:42,279 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=45953 2024-12-06T01:45:42,279 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=45953 2024-12-06T01:45:42,284 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=45953 2024-12-06T01:45:42,284 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=45953 2024-12-06T01:45:42,296 DEBUG [M:0;00c764f7c3aa:37657 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;00c764f7c3aa:37657 2024-12-06T01:45:42,297 INFO [master/00c764f7c3aa:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/00c764f7c3aa,37657,1733449542177 2024-12-06T01:45:42,299 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37657-0x1006500a2f60000, quorum=127.0.0.1:61049, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-06T01:45:42,299 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46757-0x1006500a2f60001, quorum=127.0.0.1:61049, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-06T01:45:42,299 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45953-0x1006500a2f60002, quorum=127.0.0.1:61049, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-06T01:45:42,300 DEBUG [master/00c764f7c3aa:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:37657-0x1006500a2f60000, quorum=127.0.0.1:61049, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/00c764f7c3aa,37657,1733449542177 2024-12-06T01:45:42,302 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45953-0x1006500a2f60002, quorum=127.0.0.1:61049, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-12-06T01:45:42,302 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45953-0x1006500a2f60002, quorum=127.0.0.1:61049, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-06T01:45:42,302 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46757-0x1006500a2f60001, quorum=127.0.0.1:61049, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-12-06T01:45:42,302 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37657-0x1006500a2f60000, quorum=127.0.0.1:61049, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-06T01:45:42,302 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46757-0x1006500a2f60001, quorum=127.0.0.1:61049, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-06T01:45:42,303 DEBUG [master/00c764f7c3aa:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:37657-0x1006500a2f60000, quorum=127.0.0.1:61049, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-12-06T01:45:42,303 INFO [master/00c764f7c3aa:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/00c764f7c3aa,37657,1733449542177 from backup master directory 2024-12-06T01:45:42,305 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45953-0x1006500a2f60002, quorum=127.0.0.1:61049, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-06T01:45:42,305 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46757-0x1006500a2f60001, quorum=127.0.0.1:61049, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-06T01:45:42,305 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37657-0x1006500a2f60000, quorum=127.0.0.1:61049, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/00c764f7c3aa,37657,1733449542177 2024-12-06T01:45:42,305 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37657-0x1006500a2f60000, quorum=127.0.0.1:61049, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-06T01:45:42,305 WARN [master/00c764f7c3aa:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-12-06T01:45:42,306 INFO [master/00c764f7c3aa:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=00c764f7c3aa,37657,1733449542177 2024-12-06T01:45:42,307 DEBUG [master/00c764f7c3aa:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/eff72b7a-3322-3fdb-88e1-7287cd98d0d4/e93bd390-0caa-6c00-f03a-52237a196617/hbase.id] with ID: 5294bbd8-c971-4442-aa24-39f1ed46d946 2024-12-06T01:45:42,307 DEBUG [master/00c764f7c3aa:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/eff72b7a-3322-3fdb-88e1-7287cd98d0d4/e93bd390-0caa-6c00-f03a-52237a196617/.tmp/hbase.id 2024-12-06T01:45:42,311 DEBUG [master/00c764f7c3aa:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/eff72b7a-3322-3fdb-88e1-7287cd98d0d4/e93bd390-0caa-6c00-f03a-52237a196617/.tmp/hbase.id]:[file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/eff72b7a-3322-3fdb-88e1-7287cd98d0d4/e93bd390-0caa-6c00-f03a-52237a196617/hbase.id] 2024-12-06T01:45:42,353 DEBUG [master/00c764f7c3aa:0:becomeActiveMaster {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-12-06T01:45:42,353 INFO [master/00c764f7c3aa:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem. 2024-12-06T01:45:42,355 INFO [master/00c764f7c3aa:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 1ms. 2024-12-06T01:45:42,358 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46757-0x1006500a2f60001, quorum=127.0.0.1:61049, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-06T01:45:42,358 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37657-0x1006500a2f60000, quorum=127.0.0.1:61049, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-06T01:45:42,358 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45953-0x1006500a2f60002, quorum=127.0.0.1:61049, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-06T01:45:42,363 INFO [master/00c764f7c3aa:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-06T01:45:42,365 INFO [master/00c764f7c3aa:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-12-06T01:45:42,365 INFO [master/00c764f7c3aa:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-06T01:45:42,387 INFO [master/00c764f7c3aa:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/eff72b7a-3322-3fdb-88e1-7287cd98d0d4/e93bd390-0caa-6c00-f03a-52237a196617/MasterData/data/master/store 2024-12-06T01:45:42,394 DEBUG [master/00c764f7c3aa:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-06T01:45:42,395 DEBUG [master/00c764f7c3aa:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-12-06T01:45:42,395 INFO [master/00c764f7c3aa:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-06T01:45:42,395 DEBUG [master/00c764f7c3aa:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-06T01:45:42,395 DEBUG [master/00c764f7c3aa:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-12-06T01:45:42,395 DEBUG [master/00c764f7c3aa:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-06T01:45:42,395 INFO [master/00c764f7c3aa:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-06T01:45:42,395 DEBUG [master/00c764f7c3aa:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1733449542394Disabling compacts and flushes for region at 1733449542394Disabling writes for close at 1733449542395 (+1 ms)Writing region close event to WAL at 1733449542395Closed at 1733449542395 2024-12-06T01:45:42,395 WARN [master/00c764f7c3aa:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/eff72b7a-3322-3fdb-88e1-7287cd98d0d4/e93bd390-0caa-6c00-f03a-52237a196617/MasterData/data/master/store/.initializing 2024-12-06T01:45:42,395 DEBUG [master/00c764f7c3aa:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/eff72b7a-3322-3fdb-88e1-7287cd98d0d4/e93bd390-0caa-6c00-f03a-52237a196617/MasterData/WALs/00c764f7c3aa,37657,1733449542177 2024-12-06T01:45:42,401 INFO [master/00c764f7c3aa:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-06T01:45:42,405 INFO [master/00c764f7c3aa:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=00c764f7c3aa%2C37657%2C1733449542177, suffix=, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/eff72b7a-3322-3fdb-88e1-7287cd98d0d4/e93bd390-0caa-6c00-f03a-52237a196617/MasterData/WALs/00c764f7c3aa,37657,1733449542177, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/eff72b7a-3322-3fdb-88e1-7287cd98d0d4/e93bd390-0caa-6c00-f03a-52237a196617/MasterData/oldWALs, maxLogs=10 2024-12-06T01:45:42,423 INFO [master/00c764f7c3aa:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/eff72b7a-3322-3fdb-88e1-7287cd98d0d4/e93bd390-0caa-6c00-f03a-52237a196617/MasterData/WALs/00c764f7c3aa,37657,1733449542177/00c764f7c3aa%2C37657%2C1733449542177.1733449542405 2024-12-06T01:45:42,424 DEBUG [master/00c764f7c3aa:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-12-06T01:45:42,424 DEBUG [master/00c764f7c3aa:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-12-06T01:45:42,424 DEBUG [master/00c764f7c3aa:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-06T01:45:42,424 DEBUG [master/00c764f7c3aa:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-12-06T01:45:42,424 DEBUG [master/00c764f7c3aa:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-12-06T01:45:42,425 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-12-06T01:45:42,430 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-12-06T01:45:42,430 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T01:45:42,430 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-06T01:45:42,430 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-12-06T01:45:42,434 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-12-06T01:45:42,435 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T01:45:42,435 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-06T01:45:42,435 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-12-06T01:45:42,439 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-12-06T01:45:42,439 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T01:45:42,439 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-06T01:45:42,439 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-12-06T01:45:42,443 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-12-06T01:45:42,443 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T01:45:42,443 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-06T01:45:42,443 DEBUG [master/00c764f7c3aa:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682 2024-12-06T01:45:42,444 DEBUG [master/00c764f7c3aa:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/eff72b7a-3322-3fdb-88e1-7287cd98d0d4/e93bd390-0caa-6c00-f03a-52237a196617/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-12-06T01:45:42,444 DEBUG [master/00c764f7c3aa:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/eff72b7a-3322-3fdb-88e1-7287cd98d0d4/e93bd390-0caa-6c00-f03a-52237a196617/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-12-06T01:45:42,444 DEBUG [master/00c764f7c3aa:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682 2024-12-06T01:45:42,444 DEBUG [master/00c764f7c3aa:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682 2024-12-06T01:45:42,445 DEBUG [master/00c764f7c3aa:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-12-06T01:45:42,445 DEBUG [master/00c764f7c3aa:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-12-06T01:45:42,452 DEBUG [master/00c764f7c3aa:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/eff72b7a-3322-3fdb-88e1-7287cd98d0d4/e93bd390-0caa-6c00-f03a-52237a196617/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-06T01:45:42,453 INFO [master/00c764f7c3aa:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=67098697, jitterRate=-1.5150010585784912E-4}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-12-06T01:45:42,454 DEBUG [master/00c764f7c3aa:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682: Writing region info on filesystem at 1733449542424Initializing all the Stores at 1733449542425 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733449542425Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733449542425Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733449542425Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733449542425Cleaning up temporary data from old regions at 1733449542444 (+19 ms)Region opened successfully at 1733449542454 (+10 ms) 2024-12-06T01:45:42,454 INFO [master/00c764f7c3aa:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-12-06T01:45:42,459 DEBUG [master/00c764f7c3aa:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5b3c23aa, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=00c764f7c3aa/172.17.0.2:0 2024-12-06T01:45:42,460 INFO [master/00c764f7c3aa:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating... 2024-12-06T01:45:42,460 INFO [master/00c764f7c3aa:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5 2024-12-06T01:45:42,460 INFO [master/00c764f7c3aa:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50 2024-12-06T01:45:42,461 INFO [master/00c764f7c3aa:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-12-06T01:45:42,461 INFO [master/00c764f7c3aa:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-12-06T01:45:42,461 INFO [master/00c764f7c3aa:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 0 msec 2024-12-06T01:45:42,461 INFO [master/00c764f7c3aa:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-12-06T01:45:42,463 INFO [master/00c764f7c3aa:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2024-12-06T01:45:42,464 DEBUG [master/00c764f7c3aa:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:37657-0x1006500a2f60000, quorum=127.0.0.1:61049, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error) 2024-12-06T01:45:42,466 DEBUG [master/00c764f7c3aa:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false 2024-12-06T01:45:42,466 INFO [master/00c764f7c3aa:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2024-12-06T01:45:42,467 DEBUG [master/00c764f7c3aa:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:37657-0x1006500a2f60000, quorum=127.0.0.1:61049, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error) 2024-12-06T01:45:42,468 DEBUG [master/00c764f7c3aa:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false 2024-12-06T01:45:42,469 INFO [master/00c764f7c3aa:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2024-12-06T01:45:42,470 DEBUG [master/00c764f7c3aa:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:37657-0x1006500a2f60000, quorum=127.0.0.1:61049, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error) 2024-12-06T01:45:42,471 DEBUG [master/00c764f7c3aa:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false 2024-12-06T01:45:42,472 DEBUG [master/00c764f7c3aa:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:37657-0x1006500a2f60000, quorum=127.0.0.1:61049, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error) 2024-12-06T01:45:42,473 DEBUG [master/00c764f7c3aa:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false 2024-12-06T01:45:42,476 DEBUG [master/00c764f7c3aa:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:37657-0x1006500a2f60000, quorum=127.0.0.1:61049, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error) 2024-12-06T01:45:42,477 DEBUG [master/00c764f7c3aa:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false 2024-12-06T01:45:42,479 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46757-0x1006500a2f60001, quorum=127.0.0.1:61049, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-12-06T01:45:42,479 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45953-0x1006500a2f60002, quorum=127.0.0.1:61049, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-12-06T01:45:42,479 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46757-0x1006500a2f60001, quorum=127.0.0.1:61049, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-06T01:45:42,479 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37657-0x1006500a2f60000, quorum=127.0.0.1:61049, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-12-06T01:45:42,479 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37657-0x1006500a2f60000, quorum=127.0.0.1:61049, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-06T01:45:42,479 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45953-0x1006500a2f60002, quorum=127.0.0.1:61049, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-06T01:45:42,480 INFO [master/00c764f7c3aa:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=00c764f7c3aa,37657,1733449542177, sessionid=0x1006500a2f60000, setting cluster-up flag (Was=false) 2024-12-06T01:45:42,483 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46757-0x1006500a2f60001, quorum=127.0.0.1:61049, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-06T01:45:42,483 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45953-0x1006500a2f60002, quorum=127.0.0.1:61049, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-06T01:45:42,483 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37657-0x1006500a2f60000, quorum=127.0.0.1:61049, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-06T01:45:42,489 DEBUG [master/00c764f7c3aa:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort 2024-12-06T01:45:42,491 DEBUG [master/00c764f7c3aa:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=00c764f7c3aa,37657,1733449542177 2024-12-06T01:45:42,494 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45953-0x1006500a2f60002, quorum=127.0.0.1:61049, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-06T01:45:42,494 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46757-0x1006500a2f60001, quorum=127.0.0.1:61049, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-06T01:45:42,494 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37657-0x1006500a2f60000, quorum=127.0.0.1:61049, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-06T01:45:42,499 DEBUG [master/00c764f7c3aa:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort 2024-12-06T01:45:42,501 DEBUG [master/00c764f7c3aa:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=00c764f7c3aa,37657,1733449542177 2024-12-06T01:45:42,502 INFO [master/00c764f7c3aa:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/eff72b7a-3322-3fdb-88e1-7287cd98d0d4/e93bd390-0caa-6c00-f03a-52237a196617/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again 2024-12-06T01:45:42,504 DEBUG [master/00c764f7c3aa:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta 2024-12-06T01:45:42,505 INFO [master/00c764f7c3aa:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2 2024-12-06T01:45:42,505 INFO [master/00c764f7c3aa:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2024-12-06T01:45:42,505 DEBUG [master/00c764f7c3aa:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: 00c764f7c3aa,37657,1733449542177 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2024-12-06T01:45:42,507 DEBUG [master/00c764f7c3aa:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/00c764f7c3aa:0, corePoolSize=5, maxPoolSize=5 2024-12-06T01:45:42,507 DEBUG [master/00c764f7c3aa:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/00c764f7c3aa:0, corePoolSize=5, maxPoolSize=5 2024-12-06T01:45:42,508 DEBUG [master/00c764f7c3aa:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/00c764f7c3aa:0, corePoolSize=5, maxPoolSize=5 2024-12-06T01:45:42,508 DEBUG [master/00c764f7c3aa:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/00c764f7c3aa:0, corePoolSize=5, maxPoolSize=5 2024-12-06T01:45:42,508 DEBUG [master/00c764f7c3aa:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/00c764f7c3aa:0, corePoolSize=10, maxPoolSize=10 2024-12-06T01:45:42,508 DEBUG [master/00c764f7c3aa:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/00c764f7c3aa:0, corePoolSize=1, maxPoolSize=1 2024-12-06T01:45:42,508 DEBUG [master/00c764f7c3aa:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/00c764f7c3aa:0, corePoolSize=2, maxPoolSize=2 2024-12-06T01:45:42,508 DEBUG [master/00c764f7c3aa:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/00c764f7c3aa:0, corePoolSize=1, maxPoolSize=1 2024-12-06T01:45:42,509 INFO [master/00c764f7c3aa:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1733449572509 2024-12-06T01:45:42,509 INFO [master/00c764f7c3aa:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2024-12-06T01:45:42,509 INFO [master/00c764f7c3aa:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2024-12-06T01:45:42,509 INFO [master/00c764f7c3aa:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2024-12-06T01:45:42,509 INFO [master/00c764f7c3aa:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2024-12-06T01:45:42,509 INFO [master/00c764f7c3aa:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2024-12-06T01:45:42,509 INFO [master/00c764f7c3aa:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2024-12-06T01:45:42,509 INFO [master/00c764f7c3aa:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-12-06T01:45:42,510 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta 2024-12-06T01:45:42,510 INFO [master/00c764f7c3aa:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2024-12-06T01:45:42,510 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region 2024-12-06T01:45:42,510 INFO [master/00c764f7c3aa:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2024-12-06T01:45:42,510 INFO [master/00c764f7c3aa:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2024-12-06T01:45:42,511 INFO [master/00c764f7c3aa:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2024-12-06T01:45:42,511 INFO [master/00c764f7c3aa:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2024-12-06T01:45:42,511 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T01:45:42,511 DEBUG [master/00c764f7c3aa:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/00c764f7c3aa:0:becomeActiveMaster-HFileCleaner.large.0-1733449542511,5,FailOnTimeoutGroup] 2024-12-06T01:45:42,511 DEBUG [master/00c764f7c3aa:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/00c764f7c3aa:0:becomeActiveMaster-HFileCleaner.small.0-1733449542511,5,FailOnTimeoutGroup] 2024-12-06T01:45:42,511 INFO [master/00c764f7c3aa:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-12-06T01:45:42,511 INFO [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-12-06T01:45:42,511 INFO [master/00c764f7c3aa:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2024-12-06T01:45:42,511 INFO [master/00c764f7c3aa:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2024-12-06T01:45:42,512 INFO [master/00c764f7c3aa:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2024-12-06T01:45:42,527 INFO [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/eff72b7a-3322-3fdb-88e1-7287cd98d0d4/e93bd390-0caa-6c00-f03a-52237a196617/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321 2024-12-06T01:45:42,528 INFO [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/eff72b7a-3322-3fdb-88e1-7287cd98d0d4/e93bd390-0caa-6c00-f03a-52237a196617 2024-12-06T01:45:42,539 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-06T01:45:42,540 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family info of region 1588230740 2024-12-06T01:45:42,544 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-12-06T01:45:42,544 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T01:45:42,545 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-06T01:45:42,545 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family ns of region 1588230740 2024-12-06T01:45:42,550 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-12-06T01:45:42,550 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T01:45:42,550 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-06T01:45:42,550 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family rep_barrier of region 1588230740 2024-12-06T01:45:42,554 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-12-06T01:45:42,554 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T01:45:42,554 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-06T01:45:42,555 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family table of region 1588230740 2024-12-06T01:45:42,560 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-12-06T01:45:42,560 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T01:45:42,561 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-06T01:45:42,561 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-12-06T01:45:42,562 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/eff72b7a-3322-3fdb-88e1-7287cd98d0d4/e93bd390-0caa-6c00-f03a-52237a196617/data/hbase/meta/1588230740 2024-12-06T01:45:42,562 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/eff72b7a-3322-3fdb-88e1-7287cd98d0d4/e93bd390-0caa-6c00-f03a-52237a196617/data/hbase/meta/1588230740 2024-12-06T01:45:42,562 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-12-06T01:45:42,562 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-12-06T01:45:42,563 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-12-06T01:45:42,564 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-12-06T01:45:42,571 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/eff72b7a-3322-3fdb-88e1-7287cd98d0d4/e93bd390-0caa-6c00-f03a-52237a196617/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-06T01:45:42,571 INFO [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=59950638, jitterRate=-0.10666587948799133}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-12-06T01:45:42,572 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740: Writing region info on filesystem at 1733449542539Initializing all the Stores at 1733449542539Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733449542539Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733449542539Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733449542539Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733449542540 (+1 ms)Cleaning up temporary data from old regions at 1733449542562 (+22 ms)Region opened successfully at 1733449542572 (+10 ms) 2024-12-06T01:45:42,572 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-12-06T01:45:42,572 INFO [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-12-06T01:45:42,572 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-12-06T01:45:42,572 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-12-06T01:45:42,572 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-12-06T01:45:42,573 INFO [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-12-06T01:45:42,573 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1733449542572Disabling compacts and flushes for region at 1733449542572Disabling writes for close at 1733449542572Writing region close event to WAL at 1733449542573 (+1 ms)Closed at 1733449542573 2024-12-06T01:45:42,574 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta 2024-12-06T01:45:42,574 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta 2024-12-06T01:45:42,574 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-12-06T01:45:42,575 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-12-06T01:45:42,576 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2024-12-06T01:45:42,587 INFO [RS:1;00c764f7c3aa:45953 {}] regionserver.HRegionServer(746): ClusterId : 5294bbd8-c971-4442-aa24-39f1ed46d946 2024-12-06T01:45:42,587 INFO [RS:0;00c764f7c3aa:46757 {}] regionserver.HRegionServer(746): ClusterId : 5294bbd8-c971-4442-aa24-39f1ed46d946 2024-12-06T01:45:42,587 DEBUG [RS:1;00c764f7c3aa:45953 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-12-06T01:45:42,587 DEBUG [RS:0;00c764f7c3aa:46757 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-12-06T01:45:42,590 DEBUG [RS:0;00c764f7c3aa:46757 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-12-06T01:45:42,590 DEBUG [RS:1;00c764f7c3aa:45953 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-12-06T01:45:42,590 DEBUG [RS:0;00c764f7c3aa:46757 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-12-06T01:45:42,590 DEBUG [RS:1;00c764f7c3aa:45953 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-12-06T01:45:42,593 DEBUG [RS:0;00c764f7c3aa:46757 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-12-06T01:45:42,593 DEBUG [RS:1;00c764f7c3aa:45953 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-12-06T01:45:42,594 DEBUG [RS:0;00c764f7c3aa:46757 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@275d0468, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=00c764f7c3aa/172.17.0.2:0 2024-12-06T01:45:42,594 DEBUG [RS:1;00c764f7c3aa:45953 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@64647e64, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=00c764f7c3aa/172.17.0.2:0 2024-12-06T01:45:42,614 DEBUG [RS:0;00c764f7c3aa:46757 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;00c764f7c3aa:46757 2024-12-06T01:45:42,614 DEBUG [RS:1;00c764f7c3aa:45953 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:1;00c764f7c3aa:45953 2024-12-06T01:45:42,614 INFO [RS:0;00c764f7c3aa:46757 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-12-06T01:45:42,614 INFO [RS:0;00c764f7c3aa:46757 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-12-06T01:45:42,614 INFO [RS:1;00c764f7c3aa:45953 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-12-06T01:45:42,614 DEBUG [RS:0;00c764f7c3aa:46757 {}] regionserver.HRegionServer(832): About to register with Master. 2024-12-06T01:45:42,614 INFO [RS:1;00c764f7c3aa:45953 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-12-06T01:45:42,615 DEBUG [RS:1;00c764f7c3aa:45953 {}] regionserver.HRegionServer(832): About to register with Master. 2024-12-06T01:45:42,616 INFO [RS:0;00c764f7c3aa:46757 {}] regionserver.HRegionServer(2659): reportForDuty to master=00c764f7c3aa,37657,1733449542177 with port=46757, startcode=1733449542231 2024-12-06T01:45:42,616 INFO [RS:1;00c764f7c3aa:45953 {}] regionserver.HRegionServer(2659): reportForDuty to master=00c764f7c3aa,37657,1733449542177 with port=45953, startcode=1733449542265 2024-12-06T01:45:42,616 DEBUG [RS:0;00c764f7c3aa:46757 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-12-06T01:45:42,616 DEBUG [RS:1;00c764f7c3aa:45953 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-12-06T01:45:42,619 INFO [HMaster-EventLoopGroup-6-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50479, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=RegionServerStatusService 2024-12-06T01:45:42,619 INFO [HMaster-EventLoopGroup-6-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33083, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=RegionServerStatusService 2024-12-06T01:45:42,620 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=37657 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 00c764f7c3aa,45953,1733449542265 2024-12-06T01:45:42,620 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=37657 {}] master.ServerManager(517): Registering regionserver=00c764f7c3aa,45953,1733449542265 2024-12-06T01:45:42,622 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=37657 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 00c764f7c3aa,46757,1733449542231 2024-12-06T01:45:42,622 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=37657 {}] master.ServerManager(517): Registering regionserver=00c764f7c3aa,46757,1733449542231 2024-12-06T01:45:42,622 DEBUG [RS:1;00c764f7c3aa:45953 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/eff72b7a-3322-3fdb-88e1-7287cd98d0d4/e93bd390-0caa-6c00-f03a-52237a196617 2024-12-06T01:45:42,622 DEBUG [RS:1;00c764f7c3aa:45953 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=file:/ 2024-12-06T01:45:42,622 DEBUG [RS:1;00c764f7c3aa:45953 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-12-06T01:45:42,624 DEBUG [RS:0;00c764f7c3aa:46757 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/eff72b7a-3322-3fdb-88e1-7287cd98d0d4/e93bd390-0caa-6c00-f03a-52237a196617 2024-12-06T01:45:42,624 DEBUG [RS:0;00c764f7c3aa:46757 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=file:/ 2024-12-06T01:45:42,624 DEBUG [RS:0;00c764f7c3aa:46757 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-12-06T01:45:42,625 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37657-0x1006500a2f60000, quorum=127.0.0.1:61049, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-12-06T01:45:42,628 DEBUG [RS:1;00c764f7c3aa:45953 {}] zookeeper.ZKUtil(111): regionserver:45953-0x1006500a2f60002, quorum=127.0.0.1:61049, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/00c764f7c3aa,45953,1733449542265 2024-12-06T01:45:42,628 WARN [RS:1;00c764f7c3aa:45953 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-12-06T01:45:42,628 INFO [RS:1;00c764f7c3aa:45953 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-06T01:45:42,629 DEBUG [RS:1;00c764f7c3aa:45953 {}] regionserver.HRegionServer(1793): logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/eff72b7a-3322-3fdb-88e1-7287cd98d0d4/e93bd390-0caa-6c00-f03a-52237a196617/WALs/00c764f7c3aa,45953,1733449542265 2024-12-06T01:45:42,629 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [00c764f7c3aa,46757,1733449542231] 2024-12-06T01:45:42,629 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [00c764f7c3aa,45953,1733449542265] 2024-12-06T01:45:42,629 DEBUG [RS:0;00c764f7c3aa:46757 {}] zookeeper.ZKUtil(111): regionserver:46757-0x1006500a2f60001, quorum=127.0.0.1:61049, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/00c764f7c3aa,46757,1733449542231 2024-12-06T01:45:42,629 WARN [RS:0;00c764f7c3aa:46757 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-12-06T01:45:42,629 INFO [RS:0;00c764f7c3aa:46757 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-06T01:45:42,629 DEBUG [RS:0;00c764f7c3aa:46757 {}] regionserver.HRegionServer(1793): logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/eff72b7a-3322-3fdb-88e1-7287cd98d0d4/e93bd390-0caa-6c00-f03a-52237a196617/WALs/00c764f7c3aa,46757,1733449542231 2024-12-06T01:45:42,635 INFO [RS:1;00c764f7c3aa:45953 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-12-06T01:45:42,638 INFO [RS:0;00c764f7c3aa:46757 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-12-06T01:45:42,638 INFO [RS:1;00c764f7c3aa:45953 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-12-06T01:45:42,638 INFO [RS:1;00c764f7c3aa:45953 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-12-06T01:45:42,638 INFO [RS:1;00c764f7c3aa:45953 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-12-06T01:45:42,639 INFO [RS:1;00c764f7c3aa:45953 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-12-06T01:45:42,639 INFO [RS:1;00c764f7c3aa:45953 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-12-06T01:45:42,639 INFO [RS:1;00c764f7c3aa:45953 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-12-06T01:45:42,640 DEBUG [RS:1;00c764f7c3aa:45953 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/00c764f7c3aa:0, corePoolSize=1, maxPoolSize=1 2024-12-06T01:45:42,640 DEBUG [RS:1;00c764f7c3aa:45953 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/00c764f7c3aa:0, corePoolSize=1, maxPoolSize=1 2024-12-06T01:45:42,640 DEBUG [RS:1;00c764f7c3aa:45953 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/00c764f7c3aa:0, corePoolSize=1, maxPoolSize=1 2024-12-06T01:45:42,640 DEBUG [RS:1;00c764f7c3aa:45953 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/00c764f7c3aa:0, corePoolSize=1, maxPoolSize=1 2024-12-06T01:45:42,640 DEBUG [RS:1;00c764f7c3aa:45953 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/00c764f7c3aa:0, corePoolSize=1, maxPoolSize=1 2024-12-06T01:45:42,640 DEBUG [RS:1;00c764f7c3aa:45953 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/00c764f7c3aa:0, corePoolSize=2, maxPoolSize=2 2024-12-06T01:45:42,640 INFO [RS:0;00c764f7c3aa:46757 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-12-06T01:45:42,640 DEBUG [RS:1;00c764f7c3aa:45953 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/00c764f7c3aa:0, corePoolSize=1, maxPoolSize=1 2024-12-06T01:45:42,640 DEBUG [RS:1;00c764f7c3aa:45953 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/00c764f7c3aa:0, corePoolSize=1, maxPoolSize=1 2024-12-06T01:45:42,641 DEBUG [RS:1;00c764f7c3aa:45953 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/00c764f7c3aa:0, corePoolSize=1, maxPoolSize=1 2024-12-06T01:45:42,641 INFO [RS:0;00c764f7c3aa:46757 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-12-06T01:45:42,641 DEBUG [RS:1;00c764f7c3aa:45953 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/00c764f7c3aa:0, corePoolSize=1, maxPoolSize=1 2024-12-06T01:45:42,641 INFO [RS:0;00c764f7c3aa:46757 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-12-06T01:45:42,641 DEBUG [RS:1;00c764f7c3aa:45953 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/00c764f7c3aa:0, corePoolSize=1, maxPoolSize=1 2024-12-06T01:45:42,641 DEBUG [RS:1;00c764f7c3aa:45953 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/00c764f7c3aa:0, corePoolSize=1, maxPoolSize=1 2024-12-06T01:45:42,641 INFO [RS:0;00c764f7c3aa:46757 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-12-06T01:45:42,641 DEBUG [RS:1;00c764f7c3aa:45953 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/00c764f7c3aa:0, corePoolSize=3, maxPoolSize=3 2024-12-06T01:45:42,641 DEBUG [RS:1;00c764f7c3aa:45953 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/00c764f7c3aa:0, corePoolSize=3, maxPoolSize=3 2024-12-06T01:45:42,642 INFO [RS:0;00c764f7c3aa:46757 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-12-06T01:45:42,642 INFO [RS:0;00c764f7c3aa:46757 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-12-06T01:45:42,642 DEBUG [RS:0;00c764f7c3aa:46757 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/00c764f7c3aa:0, corePoolSize=1, maxPoolSize=1 2024-12-06T01:45:42,642 DEBUG [RS:0;00c764f7c3aa:46757 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/00c764f7c3aa:0, corePoolSize=1, maxPoolSize=1 2024-12-06T01:45:42,642 DEBUG [RS:0;00c764f7c3aa:46757 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/00c764f7c3aa:0, corePoolSize=1, maxPoolSize=1 2024-12-06T01:45:42,643 DEBUG [RS:0;00c764f7c3aa:46757 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/00c764f7c3aa:0, corePoolSize=1, maxPoolSize=1 2024-12-06T01:45:42,643 DEBUG [RS:0;00c764f7c3aa:46757 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/00c764f7c3aa:0, corePoolSize=1, maxPoolSize=1 2024-12-06T01:45:42,643 DEBUG [RS:0;00c764f7c3aa:46757 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/00c764f7c3aa:0, corePoolSize=2, maxPoolSize=2 2024-12-06T01:45:42,643 DEBUG [RS:0;00c764f7c3aa:46757 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/00c764f7c3aa:0, corePoolSize=1, maxPoolSize=1 2024-12-06T01:45:42,643 DEBUG [RS:0;00c764f7c3aa:46757 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/00c764f7c3aa:0, corePoolSize=1, maxPoolSize=1 2024-12-06T01:45:42,643 DEBUG [RS:0;00c764f7c3aa:46757 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/00c764f7c3aa:0, corePoolSize=1, maxPoolSize=1 2024-12-06T01:45:42,643 DEBUG [RS:0;00c764f7c3aa:46757 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/00c764f7c3aa:0, corePoolSize=1, maxPoolSize=1 2024-12-06T01:45:42,643 DEBUG [RS:0;00c764f7c3aa:46757 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/00c764f7c3aa:0, corePoolSize=1, maxPoolSize=1 2024-12-06T01:45:42,644 DEBUG [RS:0;00c764f7c3aa:46757 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/00c764f7c3aa:0, corePoolSize=1, maxPoolSize=1 2024-12-06T01:45:42,644 DEBUG [RS:0;00c764f7c3aa:46757 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/00c764f7c3aa:0, corePoolSize=3, maxPoolSize=3 2024-12-06T01:45:42,644 DEBUG [RS:0;00c764f7c3aa:46757 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/00c764f7c3aa:0, corePoolSize=3, maxPoolSize=3 2024-12-06T01:45:42,648 INFO [RS:1;00c764f7c3aa:45953 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-12-06T01:45:42,648 INFO [RS:1;00c764f7c3aa:45953 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-12-06T01:45:42,648 INFO [RS:0;00c764f7c3aa:46757 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-12-06T01:45:42,648 INFO [RS:1;00c764f7c3aa:45953 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-06T01:45:42,648 INFO [RS:0;00c764f7c3aa:46757 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-12-06T01:45:42,648 INFO [RS:1;00c764f7c3aa:45953 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-12-06T01:45:42,649 INFO [RS:0;00c764f7c3aa:46757 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-06T01:45:42,649 INFO [RS:0;00c764f7c3aa:46757 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-12-06T01:45:42,649 INFO [RS:1;00c764f7c3aa:45953 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-12-06T01:45:42,649 INFO [RS:0;00c764f7c3aa:46757 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-12-06T01:45:42,649 INFO [RS:1;00c764f7c3aa:45953 {}] hbase.ChoreService(168): Chore ScheduledChore name=00c764f7c3aa,45953,1733449542265-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-12-06T01:45:42,649 INFO [RS:0;00c764f7c3aa:46757 {}] hbase.ChoreService(168): Chore ScheduledChore name=00c764f7c3aa,46757,1733449542231-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-12-06T01:45:42,674 INFO [RS:1;00c764f7c3aa:45953 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-12-06T01:45:42,674 INFO [RS:0;00c764f7c3aa:46757 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-12-06T01:45:42,674 INFO [RS:0;00c764f7c3aa:46757 {}] hbase.ChoreService(168): Chore ScheduledChore name=00c764f7c3aa,46757,1733449542231-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-06T01:45:42,674 INFO [RS:1;00c764f7c3aa:45953 {}] hbase.ChoreService(168): Chore ScheduledChore name=00c764f7c3aa,45953,1733449542265-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-06T01:45:42,674 INFO [RS:0;00c764f7c3aa:46757 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-12-06T01:45:42,674 INFO [RS:1;00c764f7c3aa:45953 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-12-06T01:45:42,674 INFO [RS:1;00c764f7c3aa:45953 {}] regionserver.Replication(171): 00c764f7c3aa,45953,1733449542265 started 2024-12-06T01:45:42,674 INFO [RS:0;00c764f7c3aa:46757 {}] regionserver.Replication(171): 00c764f7c3aa,46757,1733449542231 started 2024-12-06T01:45:42,699 INFO [RS:0;00c764f7c3aa:46757 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-12-06T01:45:42,699 INFO [RS:1;00c764f7c3aa:45953 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-12-06T01:45:42,699 INFO [RS:1;00c764f7c3aa:45953 {}] regionserver.HRegionServer(1482): Serving as 00c764f7c3aa,45953,1733449542265, RpcServer on 00c764f7c3aa/172.17.0.2:45953, sessionid=0x1006500a2f60002 2024-12-06T01:45:42,699 INFO [RS:0;00c764f7c3aa:46757 {}] regionserver.HRegionServer(1482): Serving as 00c764f7c3aa,46757,1733449542231, RpcServer on 00c764f7c3aa/172.17.0.2:46757, sessionid=0x1006500a2f60001 2024-12-06T01:45:42,699 DEBUG [RS:1;00c764f7c3aa:45953 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-12-06T01:45:42,699 DEBUG [RS:0;00c764f7c3aa:46757 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-12-06T01:45:42,699 DEBUG [RS:1;00c764f7c3aa:45953 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 00c764f7c3aa,45953,1733449542265 2024-12-06T01:45:42,699 DEBUG [RS:0;00c764f7c3aa:46757 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 00c764f7c3aa,46757,1733449542231 2024-12-06T01:45:42,699 DEBUG [RS:0;00c764f7c3aa:46757 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '00c764f7c3aa,46757,1733449542231' 2024-12-06T01:45:42,699 DEBUG [RS:1;00c764f7c3aa:45953 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '00c764f7c3aa,45953,1733449542265' 2024-12-06T01:45:42,699 DEBUG [RS:1;00c764f7c3aa:45953 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-12-06T01:45:42,699 DEBUG [RS:0;00c764f7c3aa:46757 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-12-06T01:45:42,700 DEBUG [RS:0;00c764f7c3aa:46757 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-12-06T01:45:42,700 DEBUG [RS:1;00c764f7c3aa:45953 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-12-06T01:45:42,701 DEBUG [RS:1;00c764f7c3aa:45953 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-12-06T01:45:42,701 DEBUG [RS:0;00c764f7c3aa:46757 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-12-06T01:45:42,701 DEBUG [RS:1;00c764f7c3aa:45953 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-12-06T01:45:42,701 DEBUG [RS:0;00c764f7c3aa:46757 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-12-06T01:45:42,701 DEBUG [RS:1;00c764f7c3aa:45953 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 00c764f7c3aa,45953,1733449542265 2024-12-06T01:45:42,701 DEBUG [RS:0;00c764f7c3aa:46757 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 00c764f7c3aa,46757,1733449542231 2024-12-06T01:45:42,701 DEBUG [RS:1;00c764f7c3aa:45953 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '00c764f7c3aa,45953,1733449542265' 2024-12-06T01:45:42,701 DEBUG [RS:0;00c764f7c3aa:46757 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '00c764f7c3aa,46757,1733449542231' 2024-12-06T01:45:42,701 DEBUG [RS:1;00c764f7c3aa:45953 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-12-06T01:45:42,701 DEBUG [RS:0;00c764f7c3aa:46757 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-12-06T01:45:42,702 DEBUG [RS:1;00c764f7c3aa:45953 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-12-06T01:45:42,702 DEBUG [RS:0;00c764f7c3aa:46757 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-12-06T01:45:42,703 DEBUG [RS:1;00c764f7c3aa:45953 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-12-06T01:45:42,703 INFO [RS:1;00c764f7c3aa:45953 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-12-06T01:45:42,703 DEBUG [RS:0;00c764f7c3aa:46757 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-12-06T01:45:42,703 INFO [RS:0;00c764f7c3aa:46757 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-12-06T01:45:42,703 INFO [RS:1;00c764f7c3aa:45953 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-12-06T01:45:42,703 INFO [RS:0;00c764f7c3aa:46757 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-12-06T01:45:42,727 WARN [00c764f7c3aa:37657 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-12-06T01:45:42,804 INFO [RS:1;00c764f7c3aa:45953 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-06T01:45:42,804 INFO [RS:0;00c764f7c3aa:46757 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-06T01:45:42,805 INFO [RS:0;00c764f7c3aa:46757 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=00c764f7c3aa%2C46757%2C1733449542231, suffix=, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/eff72b7a-3322-3fdb-88e1-7287cd98d0d4/e93bd390-0caa-6c00-f03a-52237a196617/WALs/00c764f7c3aa,46757,1733449542231, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/eff72b7a-3322-3fdb-88e1-7287cd98d0d4/e93bd390-0caa-6c00-f03a-52237a196617/oldWALs, maxLogs=55 2024-12-06T01:45:42,805 INFO [RS:1;00c764f7c3aa:45953 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=00c764f7c3aa%2C45953%2C1733449542265, suffix=, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/eff72b7a-3322-3fdb-88e1-7287cd98d0d4/e93bd390-0caa-6c00-f03a-52237a196617/WALs/00c764f7c3aa,45953,1733449542265, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/eff72b7a-3322-3fdb-88e1-7287cd98d0d4/e93bd390-0caa-6c00-f03a-52237a196617/oldWALs, maxLogs=55 2024-12-06T01:45:42,835 INFO [RS:0;00c764f7c3aa:46757 {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/eff72b7a-3322-3fdb-88e1-7287cd98d0d4/e93bd390-0caa-6c00-f03a-52237a196617/WALs/00c764f7c3aa,46757,1733449542231/00c764f7c3aa%2C46757%2C1733449542231.1733449542808 2024-12-06T01:45:42,835 INFO [RS:1;00c764f7c3aa:45953 {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/eff72b7a-3322-3fdb-88e1-7287cd98d0d4/e93bd390-0caa-6c00-f03a-52237a196617/WALs/00c764f7c3aa,45953,1733449542265/00c764f7c3aa%2C45953%2C1733449542265.1733449542808 2024-12-06T01:45:42,836 DEBUG [RS:0;00c764f7c3aa:46757 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-12-06T01:45:42,836 DEBUG [RS:1;00c764f7c3aa:45953 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-12-06T01:45:42,977 DEBUG [00c764f7c3aa:37657 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=2, allServersCount=2 2024-12-06T01:45:42,977 DEBUG [00c764f7c3aa:37657 {}] balancer.BalancerClusterState(204): Hosts are {00c764f7c3aa=0} racks are {/default-rack=0} 2024-12-06T01:45:42,978 DEBUG [00c764f7c3aa:37657 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-12-06T01:45:42,978 DEBUG [00c764f7c3aa:37657 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-12-06T01:45:42,978 DEBUG [00c764f7c3aa:37657 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-12-06T01:45:42,978 DEBUG [00c764f7c3aa:37657 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-12-06T01:45:42,978 INFO [00c764f7c3aa:37657 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-12-06T01:45:42,978 INFO [00c764f7c3aa:37657 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-12-06T01:45:42,978 DEBUG [00c764f7c3aa:37657 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-12-06T01:45:42,979 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=00c764f7c3aa,45953,1733449542265 2024-12-06T01:45:42,980 INFO [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 00c764f7c3aa,45953,1733449542265, state=OPENING 2024-12-06T01:45:42,982 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2024-12-06T01:45:42,983 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46757-0x1006500a2f60001, quorum=127.0.0.1:61049, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-06T01:45:42,984 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37657-0x1006500a2f60000, quorum=127.0.0.1:61049, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-06T01:45:42,984 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45953-0x1006500a2f60002, quorum=127.0.0.1:61049, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-06T01:45:42,984 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-06T01:45:42,984 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-06T01:45:42,984 DEBUG [PEWorker-3 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-12-06T01:45:42,985 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-06T01:45:42,985 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=00c764f7c3aa,45953,1733449542265}] 2024-12-06T01:45:43,138 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-12-06T01:45:43,141 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-8-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:32863, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-12-06T01:45:43,143 INFO [RS_OPEN_META-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-12-06T01:45:43,143 INFO [RS_OPEN_META-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-06T01:45:43,144 INFO [RS_OPEN_META-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-12-06T01:45:43,144 INFO [RS_OPEN_META-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=00c764f7c3aa%2C45953%2C1733449542265.meta, suffix=.meta, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/eff72b7a-3322-3fdb-88e1-7287cd98d0d4/e93bd390-0caa-6c00-f03a-52237a196617/WALs/00c764f7c3aa,45953,1733449542265, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/eff72b7a-3322-3fdb-88e1-7287cd98d0d4/e93bd390-0caa-6c00-f03a-52237a196617/oldWALs, maxLogs=55 2024-12-06T01:45:43,166 INFO [RS_OPEN_META-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/eff72b7a-3322-3fdb-88e1-7287cd98d0d4/e93bd390-0caa-6c00-f03a-52237a196617/WALs/00c764f7c3aa,45953,1733449542265/00c764f7c3aa%2C45953%2C1733449542265.meta.1733449543145.meta 2024-12-06T01:45:43,167 DEBUG [RS_OPEN_META-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-12-06T01:45:43,167 DEBUG [RS_OPEN_META-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-12-06T01:45:43,168 DEBUG [RS_OPEN_META-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-12-06T01:45:43,168 DEBUG [RS_OPEN_META-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-12-06T01:45:43,168 INFO [RS_OPEN_META-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-12-06T01:45:43,168 DEBUG [RS_OPEN_META-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-12-06T01:45:43,168 DEBUG [RS_OPEN_META-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-06T01:45:43,168 DEBUG [RS_OPEN_META-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-12-06T01:45:43,169 DEBUG [RS_OPEN_META-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-12-06T01:45:43,170 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family info of region 1588230740 2024-12-06T01:45:43,171 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-12-06T01:45:43,171 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T01:45:43,171 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-06T01:45:43,171 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family ns of region 1588230740 2024-12-06T01:45:43,172 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-12-06T01:45:43,172 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T01:45:43,172 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-06T01:45:43,173 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family rep_barrier of region 1588230740 2024-12-06T01:45:43,173 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-12-06T01:45:43,173 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T01:45:43,173 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-06T01:45:43,173 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family table of region 1588230740 2024-12-06T01:45:43,174 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-12-06T01:45:43,174 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T01:45:43,174 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-06T01:45:43,174 DEBUG [RS_OPEN_META-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-12-06T01:45:43,175 DEBUG [RS_OPEN_META-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/eff72b7a-3322-3fdb-88e1-7287cd98d0d4/e93bd390-0caa-6c00-f03a-52237a196617/data/hbase/meta/1588230740 2024-12-06T01:45:43,175 DEBUG [RS_OPEN_META-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/eff72b7a-3322-3fdb-88e1-7287cd98d0d4/e93bd390-0caa-6c00-f03a-52237a196617/data/hbase/meta/1588230740 2024-12-06T01:45:43,175 DEBUG [RS_OPEN_META-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-12-06T01:45:43,175 DEBUG [RS_OPEN_META-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-12-06T01:45:43,176 DEBUG [RS_OPEN_META-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-12-06T01:45:43,177 DEBUG [RS_OPEN_META-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-12-06T01:45:43,177 INFO [RS_OPEN_META-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=71065494, jitterRate=0.05895838141441345}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-12-06T01:45:43,177 DEBUG [RS_OPEN_META-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-12-06T01:45:43,178 DEBUG [RS_OPEN_META-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1733449543169Writing region info on filesystem at 1733449543169Initializing all the Stores at 1733449543169Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733449543169Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733449543170 (+1 ms)Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733449543170Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733449543170Cleaning up temporary data from old regions at 1733449543176 (+6 ms)Running coprocessor post-open hooks at 1733449543177 (+1 ms)Region opened successfully at 1733449543178 (+1 ms) 2024-12-06T01:45:43,179 INFO [RS_OPEN_META-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1733449543137 2024-12-06T01:45:43,182 DEBUG [RS_OPEN_META-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-12-06T01:45:43,182 INFO [RS_OPEN_META-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-12-06T01:45:43,182 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=00c764f7c3aa,45953,1733449542265 2024-12-06T01:45:43,183 INFO [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 00c764f7c3aa,45953,1733449542265, state=OPEN 2024-12-06T01:45:43,185 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46757-0x1006500a2f60001, quorum=127.0.0.1:61049, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-06T01:45:43,185 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45953-0x1006500a2f60002, quorum=127.0.0.1:61049, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-06T01:45:43,185 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37657-0x1006500a2f60000, quorum=127.0.0.1:61049, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-06T01:45:43,186 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-06T01:45:43,186 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=00c764f7c3aa,45953,1733449542265 2024-12-06T01:45:43,186 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-06T01:45:43,186 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-06T01:45:43,188 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2 2024-12-06T01:45:43,188 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=00c764f7c3aa,45953,1733449542265 in 201 msec 2024-12-06T01:45:43,190 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1 2024-12-06T01:45:43,190 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 614 msec 2024-12-06T01:45:43,191 DEBUG [PEWorker-2 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta 2024-12-06T01:45:43,191 INFO [PEWorker-2 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces 2024-12-06T01:45:43,195 DEBUG [PEWorker-2 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-06T01:45:43,195 DEBUG [PEWorker-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=00c764f7c3aa,45953,1733449542265, seqNum=-1] 2024-12-06T01:45:43,196 DEBUG [PEWorker-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-06T01:45:43,198 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-8-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49187, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-06T01:45:43,204 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 699 msec 2024-12-06T01:45:43,204 INFO [master/00c764f7c3aa:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1733449543204, completionTime=-1 2024-12-06T01:45:43,204 INFO [master/00c764f7c3aa:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=2; waited=0ms, expected min=2 server(s), max=2 server(s), master is running 2024-12-06T01:45:43,204 DEBUG [master/00c764f7c3aa:0:becomeActiveMaster {}] assignment.AssignmentManager(1764): Joining cluster... 2024-12-06T01:45:43,207 INFO [master/00c764f7c3aa:0:becomeActiveMaster {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-12-06T01:45:43,208 INFO [master/00c764f7c3aa:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1733449603207 2024-12-06T01:45:43,208 INFO [master/00c764f7c3aa:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1733449663208 2024-12-06T01:45:43,208 INFO [master/00c764f7c3aa:0:becomeActiveMaster {}] assignment.AssignmentManager(1783): Joined the cluster in 3 msec 2024-12-06T01:45:43,208 INFO [master/00c764f7c3aa:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=00c764f7c3aa,37657,1733449542177-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-06T01:45:43,208 INFO [master/00c764f7c3aa:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=00c764f7c3aa,37657,1733449542177-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2024-12-06T01:45:43,208 INFO [master/00c764f7c3aa:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=00c764f7c3aa,37657,1733449542177-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2024-12-06T01:45:43,209 INFO [master/00c764f7c3aa:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-00c764f7c3aa:37657, period=300000, unit=MILLISECONDS is enabled. 2024-12-06T01:45:43,209 INFO [master/00c764f7c3aa:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2024-12-06T01:45:43,209 INFO [master/00c764f7c3aa:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled. 2024-12-06T01:45:43,212 DEBUG [master/00c764f7c3aa:0.Chore.1 {}] janitor.CatalogJanitor(180): 2024-12-06T01:45:43,215 INFO [master/00c764f7c3aa:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 0.909sec 2024-12-06T01:45:43,215 INFO [master/00c764f7c3aa:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2024-12-06T01:45:43,215 INFO [master/00c764f7c3aa:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2024-12-06T01:45:43,215 INFO [master/00c764f7c3aa:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2024-12-06T01:45:43,216 INFO [master/00c764f7c3aa:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2024-12-06T01:45:43,216 INFO [master/00c764f7c3aa:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2024-12-06T01:45:43,216 INFO [master/00c764f7c3aa:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=00c764f7c3aa,37657,1733449542177-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-12-06T01:45:43,216 INFO [master/00c764f7c3aa:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=00c764f7c3aa,37657,1733449542177-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2024-12-06T01:45:43,217 DEBUG [master/00c764f7c3aa:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds 2024-12-06T01:45:43,217 INFO [master/00c764f7c3aa:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2024-12-06T01:45:43,217 INFO [master/00c764f7c3aa:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=00c764f7c3aa,37657,1733449542177-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled. 2024-12-06T01:45:43,287 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@17ad7caa, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-06T01:45:43,287 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 00c764f7c3aa,37657,-1 for getting cluster id 2024-12-06T01:45:43,287 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-06T01:45:43,290 DEBUG [HMaster-EventLoopGroup-6-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '5294bbd8-c971-4442-aa24-39f1ed46d946' 2024-12-06T01:45:43,290 DEBUG [RPCClient-NioEventLoopGroup-5-6 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-06T01:45:43,291 DEBUG [RPCClient-NioEventLoopGroup-5-6 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "5294bbd8-c971-4442-aa24-39f1ed46d946" 2024-12-06T01:45:43,291 DEBUG [RPCClient-NioEventLoopGroup-5-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@14005951, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-06T01:45:43,291 DEBUG [RPCClient-NioEventLoopGroup-5-6 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [00c764f7c3aa,37657,-1] 2024-12-06T01:45:43,291 DEBUG [RPCClient-NioEventLoopGroup-5-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-06T01:45:43,292 DEBUG [RPCClient-NioEventLoopGroup-5-6 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-06T01:45:43,293 INFO [HMaster-EventLoopGroup-6-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44544, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-06T01:45:43,295 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@45b03a5c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-06T01:45:43,295 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-06T01:45:43,297 DEBUG [RPCClient-NioEventLoopGroup-5-7 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=00c764f7c3aa,45953,1733449542265, seqNum=-1] 2024-12-06T01:45:43,297 DEBUG [RPCClient-NioEventLoopGroup-5-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-06T01:45:43,299 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-8-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33706, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-06T01:45:43,301 INFO [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=00c764f7c3aa,37657,1733449542177 2024-12-06T01:45:43,301 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-12-06T01:45:43,303 DEBUG [RPCClient-NioEventLoopGroup-5-7 {}] client.AsyncConnectionImpl(321): The fetched master address is 00c764f7c3aa,37657,1733449542177 2024-12-06T01:45:43,303 DEBUG [RPCClient-NioEventLoopGroup-5-7 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@5f00b6a 2024-12-06T01:45:43,303 DEBUG [RPCClient-NioEventLoopGroup-5-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-12-06T01:45:43,305 INFO [HMaster-EventLoopGroup-6-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44552, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-12-06T01:45:43,306 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37657 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'testBlockEvictionOnGracefulStop1733449543301', {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-06T01:45:43,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37657 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=testBlockEvictionOnGracefulStop1733449543301 2024-12-06T01:45:43,308 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=testBlockEvictionOnGracefulStop1733449543301 execute state=CREATE_TABLE_PRE_OPERATION 2024-12-06T01:45:43,308 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T01:45:43,308 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37657 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "testBlockEvictionOnGracefulStop1733449543301" procId is: 4 2024-12-06T01:45:43,309 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=testBlockEvictionOnGracefulStop1733449543301 execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-12-06T01:45:43,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37657 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-06T01:45:43,323 INFO [RegionOpenAndInit-testBlockEvictionOnGracefulStop1733449543301-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => d6945b918e4d33856aa9730069614ec9, NAME => 'testBlockEvictionOnGracefulStop1733449543301,,1733449543305.d6945b918e4d33856aa9730069614ec9.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testBlockEvictionOnGracefulStop1733449543301', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/eff72b7a-3322-3fdb-88e1-7287cd98d0d4/e93bd390-0caa-6c00-f03a-52237a196617 2024-12-06T01:45:43,330 DEBUG [RegionOpenAndInit-testBlockEvictionOnGracefulStop1733449543301-pool-0 {}] regionserver.HRegion(898): Instantiated testBlockEvictionOnGracefulStop1733449543301,,1733449543305.d6945b918e4d33856aa9730069614ec9.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-06T01:45:43,330 DEBUG [RegionOpenAndInit-testBlockEvictionOnGracefulStop1733449543301-pool-0 {}] regionserver.HRegion(1722): Closing d6945b918e4d33856aa9730069614ec9, disabling compactions & flushes 2024-12-06T01:45:43,330 INFO [RegionOpenAndInit-testBlockEvictionOnGracefulStop1733449543301-pool-0 {}] regionserver.HRegion(1755): Closing region testBlockEvictionOnGracefulStop1733449543301,,1733449543305.d6945b918e4d33856aa9730069614ec9. 2024-12-06T01:45:43,330 DEBUG [RegionOpenAndInit-testBlockEvictionOnGracefulStop1733449543301-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on testBlockEvictionOnGracefulStop1733449543301,,1733449543305.d6945b918e4d33856aa9730069614ec9. 2024-12-06T01:45:43,330 DEBUG [RegionOpenAndInit-testBlockEvictionOnGracefulStop1733449543301-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on testBlockEvictionOnGracefulStop1733449543301,,1733449543305.d6945b918e4d33856aa9730069614ec9. after waiting 0 ms 2024-12-06T01:45:43,330 DEBUG [RegionOpenAndInit-testBlockEvictionOnGracefulStop1733449543301-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region testBlockEvictionOnGracefulStop1733449543301,,1733449543305.d6945b918e4d33856aa9730069614ec9. 2024-12-06T01:45:43,330 INFO [RegionOpenAndInit-testBlockEvictionOnGracefulStop1733449543301-pool-0 {}] regionserver.HRegion(1973): Closed testBlockEvictionOnGracefulStop1733449543301,,1733449543305.d6945b918e4d33856aa9730069614ec9. 2024-12-06T01:45:43,330 DEBUG [RegionOpenAndInit-testBlockEvictionOnGracefulStop1733449543301-pool-0 {}] regionserver.HRegion(1676): Region close journal for d6945b918e4d33856aa9730069614ec9: Waiting for close lock at 1733449543330Disabling compacts and flushes for region at 1733449543330Disabling writes for close at 1733449543330Writing region close event to WAL at 1733449543330Closed at 1733449543330 2024-12-06T01:45:43,331 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=testBlockEvictionOnGracefulStop1733449543301 execute state=CREATE_TABLE_ADD_TO_META 2024-12-06T01:45:43,332 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testBlockEvictionOnGracefulStop1733449543301,,1733449543305.d6945b918e4d33856aa9730069614ec9.","families":{"info":[{"qualifier":"regioninfo","vlen":78,"tag":[],"timestamp":"1733449543331"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733449543331"}]},"ts":"1733449543331"} 2024-12-06T01:45:43,334 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-12-06T01:45:43,335 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=testBlockEvictionOnGracefulStop1733449543301 execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-12-06T01:45:43,335 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testBlockEvictionOnGracefulStop1733449543301","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733449543335"}]},"ts":"1733449543335"} 2024-12-06T01:45:43,338 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=testBlockEvictionOnGracefulStop1733449543301, state=ENABLING in hbase:meta 2024-12-06T01:45:43,338 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(204): Hosts are {00c764f7c3aa=0} racks are {/default-rack=0} 2024-12-06T01:45:43,338 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-12-06T01:45:43,338 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-12-06T01:45:43,338 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-12-06T01:45:43,338 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-12-06T01:45:43,339 INFO [PEWorker-3 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-12-06T01:45:43,339 INFO [PEWorker-3 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-12-06T01:45:43,339 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-12-06T01:45:43,339 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1733449543301, region=d6945b918e4d33856aa9730069614ec9, ASSIGN}] 2024-12-06T01:45:43,340 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1733449543301, region=d6945b918e4d33856aa9730069614ec9, ASSIGN 2024-12-06T01:45:43,341 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1733449543301, region=d6945b918e4d33856aa9730069614ec9, ASSIGN; state=OFFLINE, location=00c764f7c3aa,45953,1733449542265; forceNewPlan=false, retain=false 2024-12-06T01:45:43,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37657 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-06T01:45:43,491 INFO [00c764f7c3aa:37657 {}] balancer.BaseLoadBalancer(388): Reassigned 1 regions. 1 retained the pre-restart assignment. 2024-12-06T01:45:43,492 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=d6945b918e4d33856aa9730069614ec9, regionState=OPENING, regionLocation=00c764f7c3aa,45953,1733449542265 2024-12-06T01:45:43,495 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-8-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1733449543301, region=d6945b918e4d33856aa9730069614ec9, ASSIGN because future has completed 2024-12-06T01:45:43,496 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure d6945b918e4d33856aa9730069614ec9, server=00c764f7c3aa,45953,1733449542265}] 2024-12-06T01:45:43,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37657 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-06T01:45:43,651 INFO [RS_OPEN_REGION-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(132): Open testBlockEvictionOnGracefulStop1733449543301,,1733449543305.d6945b918e4d33856aa9730069614ec9. 2024-12-06T01:45:43,651 DEBUG [RS_OPEN_REGION-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7752): Opening region: {ENCODED => d6945b918e4d33856aa9730069614ec9, NAME => 'testBlockEvictionOnGracefulStop1733449543301,,1733449543305.d6945b918e4d33856aa9730069614ec9.', STARTKEY => '', ENDKEY => ''} 2024-12-06T01:45:43,652 DEBUG [RS_OPEN_REGION-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testBlockEvictionOnGracefulStop1733449543301 d6945b918e4d33856aa9730069614ec9 2024-12-06T01:45:43,652 DEBUG [RS_OPEN_REGION-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(898): Instantiated testBlockEvictionOnGracefulStop1733449543301,,1733449543305.d6945b918e4d33856aa9730069614ec9.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-06T01:45:43,652 DEBUG [RS_OPEN_REGION-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7794): checking encryption for d6945b918e4d33856aa9730069614ec9 2024-12-06T01:45:43,652 DEBUG [RS_OPEN_REGION-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7797): checking classloading for d6945b918e4d33856aa9730069614ec9 2024-12-06T01:45:43,653 INFO [StoreOpener-d6945b918e4d33856aa9730069614ec9-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family family of region d6945b918e4d33856aa9730069614ec9 2024-12-06T01:45:43,657 INFO [StoreOpener-d6945b918e4d33856aa9730069614ec9-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region d6945b918e4d33856aa9730069614ec9 columnFamilyName family 2024-12-06T01:45:43,657 DEBUG [StoreOpener-d6945b918e4d33856aa9730069614ec9-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T01:45:43,658 INFO [StoreOpener-d6945b918e4d33856aa9730069614ec9-1 {}] regionserver.HStore(327): Store=d6945b918e4d33856aa9730069614ec9/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-06T01:45:43,658 DEBUG [RS_OPEN_REGION-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1038): replaying wal for d6945b918e4d33856aa9730069614ec9 2024-12-06T01:45:43,659 DEBUG [RS_OPEN_REGION-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/eff72b7a-3322-3fdb-88e1-7287cd98d0d4/e93bd390-0caa-6c00-f03a-52237a196617/data/default/testBlockEvictionOnGracefulStop1733449543301/d6945b918e4d33856aa9730069614ec9 2024-12-06T01:45:43,659 DEBUG [RS_OPEN_REGION-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/eff72b7a-3322-3fdb-88e1-7287cd98d0d4/e93bd390-0caa-6c00-f03a-52237a196617/data/default/testBlockEvictionOnGracefulStop1733449543301/d6945b918e4d33856aa9730069614ec9 2024-12-06T01:45:43,659 DEBUG [RS_OPEN_REGION-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1048): stopping wal replay for d6945b918e4d33856aa9730069614ec9 2024-12-06T01:45:43,659 DEBUG [RS_OPEN_REGION-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1060): Cleaning up temporary data for d6945b918e4d33856aa9730069614ec9 2024-12-06T01:45:43,659 DEBUG [RS_OPEN_REGION-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1093): writing seq id for d6945b918e4d33856aa9730069614ec9 2024-12-06T01:45:43,666 DEBUG [RS_OPEN_REGION-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/eff72b7a-3322-3fdb-88e1-7287cd98d0d4/e93bd390-0caa-6c00-f03a-52237a196617/data/default/testBlockEvictionOnGracefulStop1733449543301/d6945b918e4d33856aa9730069614ec9/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-06T01:45:43,667 INFO [RS_OPEN_REGION-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1114): Opened d6945b918e4d33856aa9730069614ec9; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=61101226, jitterRate=-0.08952078223228455}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-06T01:45:43,667 DEBUG [RS_OPEN_REGION-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1122): Running coprocessor post-open hooks for d6945b918e4d33856aa9730069614ec9 2024-12-06T01:45:43,668 DEBUG [RS_OPEN_REGION-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1006): Region open journal for d6945b918e4d33856aa9730069614ec9: Running coprocessor pre-open hook at 1733449543652Writing region info on filesystem at 1733449543652Initializing all the Stores at 1733449543652Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733449543652Cleaning up temporary data from old regions at 1733449543659 (+7 ms)Running coprocessor post-open hooks at 1733449543667 (+8 ms)Region opened successfully at 1733449543668 (+1 ms) 2024-12-06T01:45:43,669 INFO [RS_OPEN_REGION-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2236): Post open deploy tasks for testBlockEvictionOnGracefulStop1733449543301,,1733449543305.d6945b918e4d33856aa9730069614ec9., pid=6, masterSystemTime=1733449543648 2024-12-06T01:45:43,671 DEBUG [RS_OPEN_REGION-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2266): Finished post open deploy task for testBlockEvictionOnGracefulStop1733449543301,,1733449543305.d6945b918e4d33856aa9730069614ec9. 2024-12-06T01:45:43,671 INFO [RS_OPEN_REGION-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(153): Opened testBlockEvictionOnGracefulStop1733449543301,,1733449543305.d6945b918e4d33856aa9730069614ec9. 2024-12-06T01:45:43,672 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=d6945b918e4d33856aa9730069614ec9, regionState=OPEN, openSeqNum=2, regionLocation=00c764f7c3aa,45953,1733449542265 2024-12-06T01:45:43,674 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-8-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure d6945b918e4d33856aa9730069614ec9, server=00c764f7c3aa,45953,1733449542265 because future has completed 2024-12-06T01:45:43,677 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=6, resume processing ppid=5 2024-12-06T01:45:43,677 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure d6945b918e4d33856aa9730069614ec9, server=00c764f7c3aa,45953,1733449542265 in 180 msec 2024-12-06T01:45:43,680 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=5, resume processing ppid=4 2024-12-06T01:45:43,680 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1733449543301, region=d6945b918e4d33856aa9730069614ec9, ASSIGN in 338 msec 2024-12-06T01:45:43,680 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=testBlockEvictionOnGracefulStop1733449543301 execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-12-06T01:45:43,680 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testBlockEvictionOnGracefulStop1733449543301","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733449543680"}]},"ts":"1733449543680"} 2024-12-06T01:45:43,683 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=testBlockEvictionOnGracefulStop1733449543301, state=ENABLED in hbase:meta 2024-12-06T01:45:43,683 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=testBlockEvictionOnGracefulStop1733449543301 execute state=CREATE_TABLE_POST_OPERATION 2024-12-06T01:45:43,685 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=testBlockEvictionOnGracefulStop1733449543301 in 377 msec 2024-12-06T01:45:43,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37657 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-06T01:45:43,939 INFO [RPCClient-NioEventLoopGroup-5-9 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:testBlockEvictionOnGracefulStop1733449543301 completed 2024-12-06T01:45:43,939 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table testBlockEvictionOnGracefulStop1733449543301 get assigned. Timeout = 60000ms 2024-12-06T01:45:43,940 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-12-06T01:45:43,943 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table testBlockEvictionOnGracefulStop1733449543301 assigned to meta. Checking AM states. 2024-12-06T01:45:43,943 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-12-06T01:45:43,943 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table testBlockEvictionOnGracefulStop1733449543301 assigned. 2024-12-06T01:45:43,947 DEBUG [RPCClient-NioEventLoopGroup-5-8 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testBlockEvictionOnGracefulStop1733449543301', row='row1', locateType=CURRENT is [region=testBlockEvictionOnGracefulStop1733449543301,,1733449543305.d6945b918e4d33856aa9730069614ec9., hostname=00c764f7c3aa,45953,1733449542265, seqNum=2] 2024-12-06T01:45:43,951 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing d6945b918e4d33856aa9730069614ec9 1/1 column families, dataSize=78 B heapSize=480 B 2024-12-06T01:45:43,962 DEBUG [Time-limited test {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 95052d11d88d4e2ca3db41260efa7f0e 2024-12-06T01:45:43,976 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/eff72b7a-3322-3fdb-88e1-7287cd98d0d4/e93bd390-0caa-6c00-f03a-52237a196617/data/default/testBlockEvictionOnGracefulStop1733449543301/d6945b918e4d33856aa9730069614ec9/.tmp/family/95052d11d88d4e2ca3db41260efa7f0e is 43, key is row1/family:qf1/1/Put/seqid=0 2024-12-06T01:45:43,977 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=78 B at sequenceid=6 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/eff72b7a-3322-3fdb-88e1-7287cd98d0d4/e93bd390-0caa-6c00-f03a-52237a196617/data/default/testBlockEvictionOnGracefulStop1733449543301/d6945b918e4d33856aa9730069614ec9/.tmp/family/95052d11d88d4e2ca3db41260efa7f0e 2024-12-06T01:45:43,980 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/eff72b7a-3322-3fdb-88e1-7287cd98d0d4/e93bd390-0caa-6c00-f03a-52237a196617/data/default/testBlockEvictionOnGracefulStop1733449543301/d6945b918e4d33856aa9730069614ec9/.tmp/family/95052d11d88d4e2ca3db41260efa7f0e as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/eff72b7a-3322-3fdb-88e1-7287cd98d0d4/e93bd390-0caa-6c00-f03a-52237a196617/data/default/testBlockEvictionOnGracefulStop1733449543301/d6945b918e4d33856aa9730069614ec9/family/95052d11d88d4e2ca3db41260efa7f0e 2024-12-06T01:45:43,982 DEBUG [Time-limited test {}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/eff72b7a-3322-3fdb-88e1-7287cd98d0d4/e93bd390-0caa-6c00-f03a-52237a196617/data/default/testBlockEvictionOnGracefulStop1733449543301/d6945b918e4d33856aa9730069614ec9/family/95052d11d88d4e2ca3db41260efa7f0e, delay=985 ms 2024-12-06T01:45:43,983 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/eff72b7a-3322-3fdb-88e1-7287cd98d0d4/e93bd390-0caa-6c00-f03a-52237a196617/data/default/testBlockEvictionOnGracefulStop1733449543301/d6945b918e4d33856aa9730069614ec9/family/95052d11d88d4e2ca3db41260efa7f0e, entries=2, sequenceid=6, filesize=4.9 K 2024-12-06T01:45:43,984 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~78 B/78, heapSize ~464 B/464, currentSize=0 B/0 for d6945b918e4d33856aa9730069614ec9 in 33ms, sequenceid=6, compaction requested=false 2024-12-06T01:45:43,984 DEBUG [Time-limited test {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'testBlockEvictionOnGracefulStop1733449543301' 2024-12-06T01:45:43,985 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for d6945b918e4d33856aa9730069614ec9: 2024-12-06T01:45:44,393 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-12-06T01:45:44,968 DEBUG [hfile-prefetch-1733449541608 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 95052d11d88d4e2ca3db41260efa7f0e_0. 2024-12-06T01:45:44,969 DEBUG [hfile-prefetch-1733449541608 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 95052d11d88d4e2ca3db41260efa7f0e_117. 2024-12-06T01:45:44,969 DEBUG [hfile-prefetch-1733449541608 {}] bucket.BucketCache(2281): Notifying caching completed for file file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/eff72b7a-3322-3fdb-88e1-7287cd98d0d4/e93bd390-0caa-6c00-f03a-52237a196617/data/default/testBlockEvictionOnGracefulStop1733449543301/d6945b918e4d33856aa9730069614ec9/family/95052d11d88d4e2ca3db41260efa7f0e, with total blocks 2, and data blocks 1 2024-12-06T01:45:44,969 DEBUG [hfile-prefetch-1733449541608 {}] bucket.BucketCache(2285): iterating over 0 entries in the backing map 2024-12-06T01:45:44,969 DEBUG [hfile-prefetch-1733449541608 {}] bucket.BucketCache(2308): Prefetch executor completed for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/eff72b7a-3322-3fdb-88e1-7287cd98d0d4/e93bd390-0caa-6c00-f03a-52237a196617/data/default/testBlockEvictionOnGracefulStop1733449543301/d6945b918e4d33856aa9730069614ec9/family/95052d11d88d4e2ca3db41260efa7f0e, but only 0 data blocks were cached. Total data blocks for file: 1. Checking for blocks pending cache in cache writer queue. 2024-12-06T01:45:44,969 DEBUG [hfile-prefetch-1733449541608 {}] bucket.BucketCache(2318): There are still blocks pending caching for file file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/eff72b7a-3322-3fdb-88e1-7287cd98d0d4/e93bd390-0caa-6c00-f03a-52237a196617/data/default/testBlockEvictionOnGracefulStop1733449543301/d6945b918e4d33856aa9730069614ec9/family/95052d11d88d4e2ca3db41260efa7f0e. Will sleep 100ms and try the verification again. 2024-12-06T01:45:44,976 DEBUG [bucket-cache-persister {}] bucket.BucketCachePersister(47): Cache is inconsistent, persisting to disk 2024-12-06T01:45:44,976 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1487): Thread bucket-cache-persister started persisting bucket cache to file 2024-12-06T01:45:44,976 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1494): Persist in new chunked persistence format. 2024-12-06T01:45:44,976 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1673): persistToFile: before persisting backing map size: 1, fullycachedFiles size: 0, chunkSize: 10000 2024-12-06T01:45:44,980 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1680): persistToFile: after persisting backing map size: 1, fullycachedFiles size: 0 2024-12-06T01:45:44,980 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1498): PersistToFile: after persisting backing map size: 1, fullycachedFiles size: 0, file name: bucket.persistence1733449544976 2024-12-06T01:45:44,980 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1509): Thread bucket-cache-persister finished persisting bucket cache to file, renaming 2024-12-06T01:45:44,986 INFO [Time-limited test {}] hbase.SingleProcessHBaseCluster(504): Stopping Thread[RS:1;00c764f7c3aa:45953,5,FailOnTimeoutGroup] 2024-12-06T01:45:44,986 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '00c764f7c3aa,45953,1733449542265' ***** 2024-12-06T01:45:44,986 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Stopping rs 1 2024-12-06T01:45:44,986 INFO [RS:1;00c764f7c3aa:45953 {}] bucket.BucketCache(1738): Disabling cache 2024-12-06T01:45:44,986 INFO [RS:1;00c764f7c3aa:45953 {}] bucket.BucketCache(1762): Shutdown bucket cache: IO persistent=true; path to write=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/21f224f3-974a-889b-42d4-cdd5a08d3a54/bucket.persistence 2024-12-06T01:45:44,986 INFO [Time-limited test-BucketCacheWriter-0 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-0 exiting, cacheEnabled=false 2024-12-06T01:45:44,986 INFO [Time-limited test-BucketCacheWriter-2 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-2 exiting, cacheEnabled=false 2024-12-06T01:45:44,986 INFO [Time-limited test-BucketCacheWriter-1 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-1 exiting, cacheEnabled=false 2024-12-06T01:45:44,987 INFO [RS:1;00c764f7c3aa:45953 {}] bucket.BucketCache(1768): Shutting down cache persister thread. 2024-12-06T01:45:44,987 WARN [bucket-cache-persister {}] bucket.BucketCachePersister(64): Interrupting BucketCachePersister thread. java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCachePersister.run(BucketCachePersister.java:45) ~[classes/:?] 2024-12-06T01:45:44,997 DEBUG [RS:1;00c764f7c3aa:45953 {}] bucket.BucketCache(1487): Thread RS:1;00c764f7c3aa:45953 started persisting bucket cache to file 2024-12-06T01:45:44,997 DEBUG [RS:1;00c764f7c3aa:45953 {}] bucket.BucketCache(1494): Persist in new chunked persistence format. 2024-12-06T01:45:44,997 DEBUG [RS:1;00c764f7c3aa:45953 {}] bucket.BucketCache(1673): persistToFile: before persisting backing map size: 1, fullycachedFiles size: 0, chunkSize: 10000 2024-12-06T01:45:45,001 DEBUG [RS:1;00c764f7c3aa:45953 {}] bucket.BucketCache(1680): persistToFile: after persisting backing map size: 1, fullycachedFiles size: 0 2024-12-06T01:45:45,001 DEBUG [RS:1;00c764f7c3aa:45953 {}] bucket.BucketCache(1498): PersistToFile: after persisting backing map size: 1, fullycachedFiles size: 0, file name: bucket.persistence1733449544997 2024-12-06T01:45:45,001 DEBUG [RS:1;00c764f7c3aa:45953 {}] bucket.BucketCache(1509): Thread RS:1;00c764f7c3aa:45953 finished persisting bucket cache to file, renaming 2024-12-06T01:45:45,001 INFO [RS:1;00c764f7c3aa:45953 {}] regionserver.HeapMemoryManager(220): Stopping 2024-12-06T01:45:45,001 INFO [RS:1;00c764f7c3aa:45953 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-12-06T01:45:45,001 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-12-06T01:45:45,001 INFO [RS:1;00c764f7c3aa:45953 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-12-06T01:45:45,001 INFO [RS:1;00c764f7c3aa:45953 {}] regionserver.HRegionServer(3091): Received CLOSE for d6945b918e4d33856aa9730069614ec9 2024-12-06T01:45:45,002 INFO [RS:1;00c764f7c3aa:45953 {}] regionserver.HRegionServer(959): stopping server 00c764f7c3aa,45953,1733449542265 2024-12-06T01:45:45,002 INFO [RS:1;00c764f7c3aa:45953 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-12-06T01:45:45,002 INFO [RS:1;00c764f7c3aa:45953 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:1;00c764f7c3aa:45953. 2024-12-06T01:45:45,002 INFO [RS_CLOSE_REGION-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1662): Closing region testBlockEvictionOnGracefulStop1733449543301,,1733449543305.d6945b918e4d33856aa9730069614ec9. during a graceful stop, and cache persistence is on, so setting evict on close to false. 2024-12-06T01:45:45,002 DEBUG [RS:1;00c764f7c3aa:45953 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-06T01:45:45,002 DEBUG [RS:1;00c764f7c3aa:45953 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-06T01:45:45,002 DEBUG [RS_CLOSE_REGION-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing d6945b918e4d33856aa9730069614ec9, disabling compactions & flushes 2024-12-06T01:45:45,002 INFO [RS:1;00c764f7c3aa:45953 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-12-06T01:45:45,002 INFO [RS_CLOSE_REGION-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testBlockEvictionOnGracefulStop1733449543301,,1733449543305.d6945b918e4d33856aa9730069614ec9. 2024-12-06T01:45:45,002 INFO [RS:1;00c764f7c3aa:45953 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-12-06T01:45:45,002 INFO [RS:1;00c764f7c3aa:45953 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-12-06T01:45:45,002 DEBUG [RS_CLOSE_REGION-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testBlockEvictionOnGracefulStop1733449543301,,1733449543305.d6945b918e4d33856aa9730069614ec9. 2024-12-06T01:45:45,002 INFO [RS:1;00c764f7c3aa:45953 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-12-06T01:45:45,002 DEBUG [RS_CLOSE_REGION-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testBlockEvictionOnGracefulStop1733449543301,,1733449543305.d6945b918e4d33856aa9730069614ec9. after waiting 0 ms 2024-12-06T01:45:45,002 DEBUG [RS_CLOSE_REGION-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testBlockEvictionOnGracefulStop1733449543301,,1733449543305.d6945b918e4d33856aa9730069614ec9. 2024-12-06T01:45:45,002 INFO [RS:1;00c764f7c3aa:45953 {}] regionserver.HRegionServer(1321): Waiting on 2 regions to close 2024-12-06T01:45:45,002 DEBUG [RS:1;00c764f7c3aa:45953 {}] regionserver.HRegionServer(1325): Online Regions={1588230740=hbase:meta,,1.1588230740, d6945b918e4d33856aa9730069614ec9=testBlockEvictionOnGracefulStop1733449543301,,1733449543305.d6945b918e4d33856aa9730069614ec9.} 2024-12-06T01:45:45,003 DEBUG [RS:1;00c764f7c3aa:45953 {}] regionserver.HRegionServer(1351): Waiting on 1588230740, d6945b918e4d33856aa9730069614ec9 2024-12-06T01:45:45,003 INFO [RS_CLOSE_META-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1662): Closing region hbase:meta,,1.1588230740 during a graceful stop, and cache persistence is on, so setting evict on close to false. 2024-12-06T01:45:45,003 DEBUG [RS_CLOSE_META-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-12-06T01:45:45,003 INFO [RS_CLOSE_META-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-12-06T01:45:45,003 DEBUG [RS_CLOSE_META-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-12-06T01:45:45,003 DEBUG [RS_CLOSE_META-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-12-06T01:45:45,003 DEBUG [RS_CLOSE_META-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-12-06T01:45:45,003 INFO [RS_CLOSE_META-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=1.75 KB heapSize=3.78 KB 2024-12-06T01:45:45,003 DEBUG [StoreFileCloser-d6945b918e4d33856aa9730069614ec9-family-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/eff72b7a-3322-3fdb-88e1-7287cd98d0d4/e93bd390-0caa-6c00-f03a-52237a196617/data/default/testBlockEvictionOnGracefulStop1733449543301/d6945b918e4d33856aa9730069614ec9/family/95052d11d88d4e2ca3db41260efa7f0e 2024-12-06T01:45:45,003 WARN [hfile-prefetch-1733449541608 {}] hfile.HFilePreadReader$1(158): Prefetch path=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/eff72b7a-3322-3fdb-88e1-7287cd98d0d4/e93bd390-0caa-6c00-f03a-52237a196617/data/default/testBlockEvictionOnGracefulStop1733449543301/d6945b918e4d33856aa9730069614ec9/family/95052d11d88d4e2ca3db41260efa7f0e, offset=158, end=158 java.lang.RuntimeException: java.lang.InterruptedException: sleep interrupted at org.apache.hadoop.hbase.io.hfile.bucket.BucketCache.notifyFileCachingCompleted(BucketCache.java:2337) ~[classes/:?] at org.apache.hadoop.hbase.io.hfile.CombinedBlockCache.notifyFileCachingCompleted(CombinedBlockCache.java:469) ~[classes/:?] at org.apache.hadoop.hbase.io.hfile.HFilePreadReader$1.run(HFilePreadReader.java:148) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.lambda$tracedRunnable$2(TraceUtil.java:155) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:539) ~[?:?] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:304) ~[?:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCache.notifyFileCachingCompleted(BucketCache.java:2320) ~[classes/:?] ... 9 more 2024-12-06T01:45:45,004 DEBUG [hfile-prefetch-1733449541608 {}] hfile.PrefetchExecutor(121): Prefetch completed for 95052d11d88d4e2ca3db41260efa7f0e 2024-12-06T01:45:45,009 DEBUG [RS_CLOSE_REGION-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/eff72b7a-3322-3fdb-88e1-7287cd98d0d4/e93bd390-0caa-6c00-f03a-52237a196617/data/default/testBlockEvictionOnGracefulStop1733449543301/d6945b918e4d33856aa9730069614ec9/recovered.edits/9.seqid, newMaxSeqId=9, maxSeqId=1 2024-12-06T01:45:45,009 INFO [RS_CLOSE_REGION-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testBlockEvictionOnGracefulStop1733449543301,,1733449543305.d6945b918e4d33856aa9730069614ec9. 2024-12-06T01:45:45,009 DEBUG [RS_CLOSE_REGION-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for d6945b918e4d33856aa9730069614ec9: Waiting for close lock at 1733449545002Running coprocessor pre-close hooks at 1733449545002Disabling compacts and flushes for region at 1733449545002Disabling writes for close at 1733449545002Writing region close event to WAL at 1733449545003 (+1 ms)Running coprocessor post-close hooks at 1733449545009 (+6 ms)Closed at 1733449545009 2024-12-06T01:45:45,009 DEBUG [RS_CLOSE_REGION-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testBlockEvictionOnGracefulStop1733449543301,,1733449543305.d6945b918e4d33856aa9730069614ec9. 2024-12-06T01:45:45,014 DEBUG [RS_CLOSE_META-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 069acf6dc56e40a29632095fab5c92df 2024-12-06T01:45:45,028 DEBUG [RS_CLOSE_META-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/eff72b7a-3322-3fdb-88e1-7287cd98d0d4/e93bd390-0caa-6c00-f03a-52237a196617/data/hbase/meta/1588230740/.tmp/info/069acf6dc56e40a29632095fab5c92df is 209, key is testBlockEvictionOnGracefulStop1733449543301,,1733449543305.d6945b918e4d33856aa9730069614ec9./info:regioninfo/1733449543672/Put/seqid=0 2024-12-06T01:45:45,029 INFO [RS_CLOSE_META-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.53 KB at sequenceid=11 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/eff72b7a-3322-3fdb-88e1-7287cd98d0d4/e93bd390-0caa-6c00-f03a-52237a196617/data/hbase/meta/1588230740/.tmp/info/069acf6dc56e40a29632095fab5c92df 2024-12-06T01:45:45,039 DEBUG [RS_CLOSE_META-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 21031730d1ed41a68fd5a6cef21a2730 2024-12-06T01:45:45,051 INFO [regionserver/00c764f7c3aa:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-12-06T01:45:45,053 DEBUG [RS_CLOSE_META-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/eff72b7a-3322-3fdb-88e1-7287cd98d0d4/e93bd390-0caa-6c00-f03a-52237a196617/data/hbase/meta/1588230740/.tmp/ns/21031730d1ed41a68fd5a6cef21a2730 is 43, key is default/ns:d/1733449543199/Put/seqid=0 2024-12-06T01:45:45,054 INFO [RS_CLOSE_META-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=74 B at sequenceid=11 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/eff72b7a-3322-3fdb-88e1-7287cd98d0d4/e93bd390-0caa-6c00-f03a-52237a196617/data/hbase/meta/1588230740/.tmp/ns/21031730d1ed41a68fd5a6cef21a2730 2024-12-06T01:45:45,062 DEBUG [RS_CLOSE_META-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 219cd3ddf070484aaebb17561edcedaf 2024-12-06T01:45:45,075 DEBUG [RS_CLOSE_META-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/eff72b7a-3322-3fdb-88e1-7287cd98d0d4/e93bd390-0caa-6c00-f03a-52237a196617/data/hbase/meta/1588230740/.tmp/table/219cd3ddf070484aaebb17561edcedaf is 80, key is testBlockEvictionOnGracefulStop1733449543301/table:state/1733449543680/Put/seqid=0 2024-12-06T01:45:45,076 INFO [RS_CLOSE_META-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=152 B at sequenceid=11 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/eff72b7a-3322-3fdb-88e1-7287cd98d0d4/e93bd390-0caa-6c00-f03a-52237a196617/data/hbase/meta/1588230740/.tmp/table/219cd3ddf070484aaebb17561edcedaf 2024-12-06T01:45:45,079 DEBUG [RS_CLOSE_META-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/eff72b7a-3322-3fdb-88e1-7287cd98d0d4/e93bd390-0caa-6c00-f03a-52237a196617/data/hbase/meta/1588230740/.tmp/info/069acf6dc56e40a29632095fab5c92df as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/eff72b7a-3322-3fdb-88e1-7287cd98d0d4/e93bd390-0caa-6c00-f03a-52237a196617/data/hbase/meta/1588230740/info/069acf6dc56e40a29632095fab5c92df 2024-12-06T01:45:45,081 DEBUG [RS_CLOSE_META-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_CLOSE_META}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/eff72b7a-3322-3fdb-88e1-7287cd98d0d4/e93bd390-0caa-6c00-f03a-52237a196617/data/hbase/meta/1588230740/info/069acf6dc56e40a29632095fab5c92df, delay=901 ms 2024-12-06T01:45:45,082 INFO [RS_CLOSE_META-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/eff72b7a-3322-3fdb-88e1-7287cd98d0d4/e93bd390-0caa-6c00-f03a-52237a196617/data/hbase/meta/1588230740/info/069acf6dc56e40a29632095fab5c92df, entries=10, sequenceid=11, filesize=7.0 K 2024-12-06T01:45:45,083 DEBUG [RS_CLOSE_META-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/eff72b7a-3322-3fdb-88e1-7287cd98d0d4/e93bd390-0caa-6c00-f03a-52237a196617/data/hbase/meta/1588230740/.tmp/ns/21031730d1ed41a68fd5a6cef21a2730 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/eff72b7a-3322-3fdb-88e1-7287cd98d0d4/e93bd390-0caa-6c00-f03a-52237a196617/data/hbase/meta/1588230740/ns/21031730d1ed41a68fd5a6cef21a2730 2024-12-06T01:45:45,084 DEBUG [RS_CLOSE_META-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_CLOSE_META}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/eff72b7a-3322-3fdb-88e1-7287cd98d0d4/e93bd390-0caa-6c00-f03a-52237a196617/data/hbase/meta/1588230740/ns/21031730d1ed41a68fd5a6cef21a2730, delay=979 ms 2024-12-06T01:45:45,086 INFO [RS_CLOSE_META-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/eff72b7a-3322-3fdb-88e1-7287cd98d0d4/e93bd390-0caa-6c00-f03a-52237a196617/data/hbase/meta/1588230740/ns/21031730d1ed41a68fd5a6cef21a2730, entries=2, sequenceid=11, filesize=5.0 K 2024-12-06T01:45:45,086 DEBUG [RS_CLOSE_META-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/eff72b7a-3322-3fdb-88e1-7287cd98d0d4/e93bd390-0caa-6c00-f03a-52237a196617/data/hbase/meta/1588230740/.tmp/table/219cd3ddf070484aaebb17561edcedaf as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/eff72b7a-3322-3fdb-88e1-7287cd98d0d4/e93bd390-0caa-6c00-f03a-52237a196617/data/hbase/meta/1588230740/table/219cd3ddf070484aaebb17561edcedaf 2024-12-06T01:45:45,087 DEBUG [RS_CLOSE_META-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_CLOSE_META}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/eff72b7a-3322-3fdb-88e1-7287cd98d0d4/e93bd390-0caa-6c00-f03a-52237a196617/data/hbase/meta/1588230740/table/219cd3ddf070484aaebb17561edcedaf, delay=983 ms 2024-12-06T01:45:45,089 INFO [RS_CLOSE_META-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/eff72b7a-3322-3fdb-88e1-7287cd98d0d4/e93bd390-0caa-6c00-f03a-52237a196617/data/hbase/meta/1588230740/table/219cd3ddf070484aaebb17561edcedaf, entries=2, sequenceid=11, filesize=5.3 K 2024-12-06T01:45:45,089 INFO [RS_CLOSE_META-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~1.75 KB/1797, heapSize ~3.48 KB/3568, currentSize=0 B/0 for 1588230740 in 86ms, sequenceid=11, compaction requested=false 2024-12-06T01:45:45,090 DEBUG [StoreFileCloser-1588230740-info-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/eff72b7a-3322-3fdb-88e1-7287cd98d0d4/e93bd390-0caa-6c00-f03a-52237a196617/data/hbase/meta/1588230740/info/069acf6dc56e40a29632095fab5c92df 2024-12-06T01:45:45,091 DEBUG [StoreFileCloser-1588230740-ns-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/eff72b7a-3322-3fdb-88e1-7287cd98d0d4/e93bd390-0caa-6c00-f03a-52237a196617/data/hbase/meta/1588230740/ns/21031730d1ed41a68fd5a6cef21a2730 2024-12-06T01:45:45,091 DEBUG [StoreFileCloser-1588230740-table-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/eff72b7a-3322-3fdb-88e1-7287cd98d0d4/e93bd390-0caa-6c00-f03a-52237a196617/data/hbase/meta/1588230740/table/219cd3ddf070484aaebb17561edcedaf 2024-12-06T01:45:45,096 DEBUG [RS_CLOSE_META-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/eff72b7a-3322-3fdb-88e1-7287cd98d0d4/e93bd390-0caa-6c00-f03a-52237a196617/data/hbase/meta/1588230740/recovered.edits/14.seqid, newMaxSeqId=14, maxSeqId=1 2024-12-06T01:45:45,097 DEBUG [RS_CLOSE_META-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-12-06T01:45:45,097 INFO [RS_CLOSE_META-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-12-06T01:45:45,097 DEBUG [RS_CLOSE_META-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1733449545003Running coprocessor pre-close hooks at 1733449545003Disabling compacts and flushes for region at 1733449545003Disabling writes for close at 1733449545003Obtaining lock to block concurrent updates at 1733449545003Preparing flush snapshotting stores in 1588230740 at 1733449545003Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=1797, getHeapSize=3808, getOffHeapSize=0, getCellsCount=14 at 1733449545004 (+1 ms)Flushing stores of hbase:meta,,1.1588230740 at 1733449545004Flushing 1588230740/info: creating writer at 1733449545004Flushing 1588230740/info: appending metadata at 1733449545028 (+24 ms)Flushing 1588230740/info: closing flushed file at 1733449545028Flushing 1588230740/ns: creating writer at 1733449545033 (+5 ms)Flushing 1588230740/ns: appending metadata at 1733449545053 (+20 ms)Flushing 1588230740/ns: closing flushed file at 1733449545053Flushing 1588230740/table: creating writer at 1733449545056 (+3 ms)Flushing 1588230740/table: appending metadata at 1733449545075 (+19 ms)Flushing 1588230740/table: closing flushed file at 1733449545075Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@7bf57ecc: reopening flushed file at 1733449545079 (+4 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@71a79d86: reopening flushed file at 1733449545082 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@772f4521: reopening flushed file at 1733449545086 (+4 ms)Finished flush of dataSize ~1.75 KB/1797, heapSize ~3.48 KB/3568, currentSize=0 B/0 for 1588230740 in 86ms, sequenceid=11, compaction requested=false at 1733449545090 (+4 ms)Writing region close event to WAL at 1733449545092 (+2 ms)Running coprocessor post-close hooks at 1733449545097 (+5 ms)Closed at 1733449545097 2024-12-06T01:45:45,097 DEBUG [RS_CLOSE_META-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-12-06T01:45:45,203 INFO [RS:1;00c764f7c3aa:45953 {}] regionserver.HRegionServer(976): stopping server 00c764f7c3aa,45953,1733449542265; all regions closed. 2024-12-06T01:45:45,205 DEBUG [RS:1;00c764f7c3aa:45953 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/eff72b7a-3322-3fdb-88e1-7287cd98d0d4/e93bd390-0caa-6c00-f03a-52237a196617/oldWALs 2024-12-06T01:45:45,206 INFO [RS:1;00c764f7c3aa:45953 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 00c764f7c3aa%2C45953%2C1733449542265.meta:.meta(num 1733449543145) 2024-12-06T01:45:45,207 DEBUG [RS:1;00c764f7c3aa:45953 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/eff72b7a-3322-3fdb-88e1-7287cd98d0d4/e93bd390-0caa-6c00-f03a-52237a196617/oldWALs 2024-12-06T01:45:45,207 INFO [RS:1;00c764f7c3aa:45953 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 00c764f7c3aa%2C45953%2C1733449542265:(num 1733449542808) 2024-12-06T01:45:45,207 DEBUG [RS:1;00c764f7c3aa:45953 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-06T01:45:45,207 INFO [RS:1;00c764f7c3aa:45953 {}] regionserver.LeaseManager(133): Closed leases 2024-12-06T01:45:45,207 INFO [RS:1;00c764f7c3aa:45953 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-12-06T01:45:45,208 INFO [RS:1;00c764f7c3aa:45953 {}] hbase.ChoreService(370): Chore service for: regionserver/00c764f7c3aa:0 had [ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-12-06T01:45:45,208 INFO [RS:1;00c764f7c3aa:45953 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-12-06T01:45:45,208 INFO [regionserver/00c764f7c3aa:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-12-06T01:45:45,208 INFO [RS:1;00c764f7c3aa:45953 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:45953 2024-12-06T01:45:45,212 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45953-0x1006500a2f60002, quorum=127.0.0.1:61049, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/00c764f7c3aa,45953,1733449542265 2024-12-06T01:45:45,212 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37657-0x1006500a2f60000, quorum=127.0.0.1:61049, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-12-06T01:45:45,213 INFO [RS:1;00c764f7c3aa:45953 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-12-06T01:45:45,215 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [00c764f7c3aa,45953,1733449542265] 2024-12-06T01:45:45,216 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/00c764f7c3aa,45953,1733449542265 already deleted, retry=false 2024-12-06T01:45:45,216 INFO [RegionServerTracker-0 {}] master.ServerManager(695): Processing expiration of 00c764f7c3aa,45953,1733449542265 on 00c764f7c3aa,37657,1733449542177 2024-12-06T01:45:45,221 DEBUG [RegionServerTracker-0 {}] procedure2.ProcedureExecutor(1139): Stored pid=7, state=RUNNABLE:SERVER_CRASH_START, hasLock=false; ServerCrashProcedure 00c764f7c3aa,45953,1733449542265, splitWal=true, meta=true 2024-12-06T01:45:45,224 INFO [PEWorker-2 {}] procedure.ServerCrashProcedure(169): Start pid=7, state=RUNNABLE:SERVER_CRASH_START, hasLock=true; ServerCrashProcedure 00c764f7c3aa,45953,1733449542265, splitWal=true, meta=true 2024-12-06T01:45:45,224 INFO [RegionServerTracker-0 {}] assignment.AssignmentManager(1999): Scheduled ServerCrashProcedure pid=7 for 00c764f7c3aa,45953,1733449542265 (carryingMeta=true) 00c764f7c3aa,45953,1733449542265/CRASHED/regionCount=2/lock=java.util.concurrent.locks.ReentrantReadWriteLock@4b6be237[Write locks = 1, Read locks = 0], oldState=ONLINE. 2024-12-06T01:45:45,225 INFO [PEWorker-2 {}] procedure.ServerCrashProcedure(339): Splitting WALs pid=7, state=RUNNABLE:SERVER_CRASH_SPLIT_META_LOGS, hasLock=true; ServerCrashProcedure 00c764f7c3aa,45953,1733449542265, splitWal=true, meta=true, isMeta: true 2024-12-06T01:45:45,225 DEBUG [PEWorker-2 {}] master.MasterWalManager(329): Renamed region directory: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/eff72b7a-3322-3fdb-88e1-7287cd98d0d4/e93bd390-0caa-6c00-f03a-52237a196617/WALs/00c764f7c3aa,45953,1733449542265-splitting 2024-12-06T01:45:45,226 INFO [PEWorker-2 {}] master.SplitLogManager(171): file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/eff72b7a-3322-3fdb-88e1-7287cd98d0d4/e93bd390-0caa-6c00-f03a-52237a196617/WALs/00c764f7c3aa,45953,1733449542265-splitting dir is empty, no logs to split. 2024-12-06T01:45:45,226 INFO [PEWorker-2 {}] master.SplitWALManager(105): 00c764f7c3aa,45953,1733449542265 WAL count=0, meta=true 2024-12-06T01:45:45,228 INFO [PEWorker-2 {}] master.SplitLogManager(171): file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/eff72b7a-3322-3fdb-88e1-7287cd98d0d4/e93bd390-0caa-6c00-f03a-52237a196617/WALs/00c764f7c3aa,45953,1733449542265-splitting dir is empty, no logs to split. 2024-12-06T01:45:45,228 INFO [PEWorker-2 {}] master.SplitWALManager(105): 00c764f7c3aa,45953,1733449542265 WAL count=0, meta=true 2024-12-06T01:45:45,228 DEBUG [PEWorker-2 {}] procedure.ServerCrashProcedure(329): Check if 00c764f7c3aa,45953,1733449542265 WAL splitting is done? wals=0, meta=true 2024-12-06T01:45:45,230 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=8, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-12-06T01:45:45,231 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=8, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-12-06T01:45:45,232 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(269): Starting pid=8, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-12-06T01:45:45,315 INFO [RS:1;00c764f7c3aa:45953 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-12-06T01:45:45,315 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45953-0x1006500a2f60002, quorum=127.0.0.1:61049, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-06T01:45:45,315 INFO [RS:1;00c764f7c3aa:45953 {}] regionserver.HRegionServer(1031): Exiting; stopping=00c764f7c3aa,45953,1733449542265; zookeeper connection closed. 2024-12-06T01:45:45,315 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45953-0x1006500a2f60002, quorum=127.0.0.1:61049, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-06T01:45:45,315 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@482cde71 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@482cde71 2024-12-06T01:45:45,382 DEBUG [00c764f7c3aa:37657 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=1, allServersCount=1 2024-12-06T01:45:45,383 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=8 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=00c764f7c3aa,46757,1733449542231 2024-12-06T01:45:45,384 INFO [PEWorker-4 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 00c764f7c3aa,46757,1733449542231, state=OPENING 2024-12-06T01:45:45,386 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46757-0x1006500a2f60001, quorum=127.0.0.1:61049, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-06T01:45:45,386 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37657-0x1006500a2f60000, quorum=127.0.0.1:61049, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-06T01:45:45,386 DEBUG [PEWorker-4 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=8, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-12-06T01:45:45,386 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-06T01:45:45,386 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-06T01:45:45,386 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=9, ppid=8, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=00c764f7c3aa,46757,1733449542231}] 2024-12-06T01:45:45,505 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/00c764f7c3aa:0 server-side Connection retries=45 2024-12-06T01:45:45,506 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-06T01:45:45,506 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-12-06T01:45:45,506 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-12-06T01:45:45,506 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-06T01:45:45,506 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-12-06T01:45:45,506 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-12-06T01:45:45,506 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-12-06T01:45:45,507 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:36715 2024-12-06T01:45:45,509 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:36715 connecting to ZooKeeper ensemble=127.0.0.1:61049 2024-12-06T01:45:45,509 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-12-06T01:45:45,509 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-12-06T01:45:45,512 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:367150x0, quorum=127.0.0.1:61049, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-06T01:45:45,513 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:36715-0x1006500a2f60003 connected 2024-12-06T01:45:45,513 DEBUG [Time-limited test {}] zookeeper.ZKUtil(111): regionserver:36715-0x1006500a2f60003, quorum=127.0.0.1:61049, baseZNode=/hbase Set watcher on existing znode=/hbase/running 2024-12-06T01:45:45,513 DEBUG [pool-298-thread-1 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: INIT 2024-12-06T01:45:45,514 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-12-06T01:45:45,514 INFO [Time-limited test {}] bucket.FileIOEngine(99): Allocating cache 400 MB, on the path:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/21f224f3-974a-889b-42d4-cdd5a08d3a54/bucket.cache 2024-12-06T01:45:45,514 INFO [Time-limited test {}] bucket.BucketCache(364): Instantiating BucketCache with acceptableFactor: 0.95, minFactor: 0.85, extraFreeFactor: 0.1, singleFactor: 0.25, multiFactor: 0.5, memoryFactor: 0.25, useStrongRef: false 2024-12-06T01:45:45,514 INFO [Time-limited test {}] bucket.BucketCache(478): Starting BucketCachePersisterThread 2024-12-06T01:45:45,514 INFO [Time-limited test {}] bucket.BucketCachePersister(38): BucketCachePersister started with interval: 100 2024-12-06T01:45:45,516 INFO [Thread-207 {}] bucket.BucketCache(1530): Started retrieving bucket cache from file 2024-12-06T01:45:45,517 INFO [Thread-207 {}] bucket.BucketCache(1552): Reading new chunked format of persistence. 2024-12-06T01:45:45,518 INFO [Time-limited test {}] bucket.BucketCache(403): Started bucket cache; ioengine=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/21f224f3-974a-889b-42d4-cdd5a08d3a54/bucket.cache, capacity=400 MB, blockSize=64 KB, writerThreadNum=3, writerQLen=64, persistencePath=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/21f224f3-974a-889b-42d4-cdd5a08d3a54/bucket.persistence, bucketAllocator=org.apache.hadoop.hbase.io.hfile.bucket.BucketAllocator 2024-12-06T01:45:45,518 INFO [Thread-207 {}] bucket.BucketCache(1705): Retrieved 1 of chunks with blockCount = 1. 2024-12-06T01:45:45,518 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-12-06T01:45:45,519 DEBUG [Time-limited test {}] zookeeper.ZKUtil(111): regionserver:36715-0x1006500a2f60003, quorum=127.0.0.1:61049, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-12-06T01:45:45,521 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:36715-0x1006500a2f60003, quorum=127.0.0.1:61049, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-12-06T01:45:45,522 DEBUG [Thread-207 {}] bucket.BucketCache(1569): Updating region size map after retrieving cached file list 2024-12-06T01:45:45,522 INFO [Thread-207 {}] bucket.BucketAllocator(343): Cache totalSize=418148352, buckets=199, bucket capacity=2101248=(4*525312)=(FEWEST_ITEMS_IN_BUCKET*(largest configured bucketcache size)) 2024-12-06T01:45:45,522 INFO [Thread-207 {}] bucket.BucketCache(1562): Bucket cache retrieved from file successfully with size: 1 2024-12-06T01:45:45,522 INFO [Thread-207 {}] bucket.BucketCache(413): Persistent bucket cache recovery from /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/21f224f3-974a-889b-42d4-cdd5a08d3a54/bucket.persistence is complete. 2024-12-06T01:45:45,531 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=36715 2024-12-06T01:45:45,532 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=36715 2024-12-06T01:45:45,532 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=36715 2024-12-06T01:45:45,532 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=36715 2024-12-06T01:45:45,533 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=36715 2024-12-06T01:45:45,537 INFO [RS:2;00c764f7c3aa:36715 {}] regionserver.HRegionServer(746): ClusterId : 5294bbd8-c971-4442-aa24-39f1ed46d946 2024-12-06T01:45:45,537 DEBUG [RS:2;00c764f7c3aa:36715 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-12-06T01:45:45,539 DEBUG [RSProcedureDispatcher-pool-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-12-06T01:45:45,540 DEBUG [RS:2;00c764f7c3aa:36715 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-12-06T01:45:45,540 DEBUG [RS:2;00c764f7c3aa:36715 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-12-06T01:45:45,541 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-7-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48859, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-12-06T01:45:45,543 DEBUG [RS:2;00c764f7c3aa:36715 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-12-06T01:45:45,543 INFO [RS_OPEN_META-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_OPEN_META, pid=9}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-12-06T01:45:45,543 INFO [RS_OPEN_META-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_OPEN_META, pid=9}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-06T01:45:45,543 DEBUG [RS:2;00c764f7c3aa:36715 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5dc4ff7c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=00c764f7c3aa/172.17.0.2:0 2024-12-06T01:45:45,544 INFO [RS_OPEN_META-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_OPEN_META, pid=9}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-12-06T01:45:45,544 INFO [RS_OPEN_META-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_OPEN_META, pid=9}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=00c764f7c3aa%2C46757%2C1733449542231.meta, suffix=.meta, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/eff72b7a-3322-3fdb-88e1-7287cd98d0d4/e93bd390-0caa-6c00-f03a-52237a196617/WALs/00c764f7c3aa,46757,1733449542231, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/eff72b7a-3322-3fdb-88e1-7287cd98d0d4/e93bd390-0caa-6c00-f03a-52237a196617/oldWALs, maxLogs=55 2024-12-06T01:45:45,565 INFO [RS_OPEN_META-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_OPEN_META, pid=9}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/eff72b7a-3322-3fdb-88e1-7287cd98d0d4/e93bd390-0caa-6c00-f03a-52237a196617/WALs/00c764f7c3aa,46757,1733449542231/00c764f7c3aa%2C46757%2C1733449542231.meta.1733449545545.meta 2024-12-06T01:45:45,565 DEBUG [RS_OPEN_META-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_OPEN_META, pid=9}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-12-06T01:45:45,566 DEBUG [RS_OPEN_META-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-12-06T01:45:45,566 DEBUG [RS_OPEN_META-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_OPEN_META, pid=9}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-12-06T01:45:45,566 DEBUG [RS_OPEN_META-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-12-06T01:45:45,566 INFO [RS_OPEN_META-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-12-06T01:45:45,567 DEBUG [RS_OPEN_META-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-12-06T01:45:45,567 DEBUG [RS_OPEN_META-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-06T01:45:45,567 DEBUG [RS:2;00c764f7c3aa:36715 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:2;00c764f7c3aa:36715 2024-12-06T01:45:45,567 DEBUG [RS_OPEN_META-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-12-06T01:45:45,567 INFO [RS:2;00c764f7c3aa:36715 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-12-06T01:45:45,567 DEBUG [RS_OPEN_META-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-12-06T01:45:45,567 INFO [RS:2;00c764f7c3aa:36715 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-12-06T01:45:45,567 DEBUG [RS:2;00c764f7c3aa:36715 {}] regionserver.HRegionServer(832): About to register with Master. 2024-12-06T01:45:45,568 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family info of region 1588230740 2024-12-06T01:45:45,568 INFO [RS:2;00c764f7c3aa:36715 {}] regionserver.HRegionServer(2659): reportForDuty to master=00c764f7c3aa,37657,1733449542177 with port=36715, startcode=1733449545505 2024-12-06T01:45:45,568 DEBUG [RS:2;00c764f7c3aa:36715 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-12-06T01:45:45,569 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-12-06T01:45:45,569 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T01:45:45,573 DEBUG [StoreFileOpener-1588230740-info-1 {}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/eff72b7a-3322-3fdb-88e1-7287cd98d0d4/e93bd390-0caa-6c00-f03a-52237a196617/data/hbase/meta/1588230740/info/069acf6dc56e40a29632095fab5c92df, delay=913 ms 2024-12-06T01:45:45,574 INFO [HMaster-EventLoopGroup-6-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36937, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=RegionServerStatusService 2024-12-06T01:45:45,574 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=37657 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 00c764f7c3aa,36715,1733449545505 2024-12-06T01:45:45,574 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=37657 {}] master.ServerManager(517): Registering regionserver=00c764f7c3aa,36715,1733449545505 2024-12-06T01:45:45,575 DEBUG [StoreOpener-1588230740-1 {}] regionserver.StoreEngine(278): loaded file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/eff72b7a-3322-3fdb-88e1-7287cd98d0d4/e93bd390-0caa-6c00-f03a-52237a196617/data/hbase/meta/1588230740/info/069acf6dc56e40a29632095fab5c92df 2024-12-06T01:45:45,575 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-06T01:45:45,575 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family ns of region 1588230740 2024-12-06T01:45:45,576 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-12-06T01:45:45,576 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T01:45:45,576 DEBUG [RS:2;00c764f7c3aa:36715 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/eff72b7a-3322-3fdb-88e1-7287cd98d0d4/e93bd390-0caa-6c00-f03a-52237a196617 2024-12-06T01:45:45,576 DEBUG [RS:2;00c764f7c3aa:36715 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=file:/ 2024-12-06T01:45:45,576 DEBUG [RS:2;00c764f7c3aa:36715 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-12-06T01:45:45,578 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37657-0x1006500a2f60000, quorum=127.0.0.1:61049, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-12-06T01:45:45,579 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [00c764f7c3aa,36715,1733449545505] 2024-12-06T01:45:45,580 DEBUG [StoreFileOpener-1588230740-ns-1 {}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/eff72b7a-3322-3fdb-88e1-7287cd98d0d4/e93bd390-0caa-6c00-f03a-52237a196617/data/hbase/meta/1588230740/ns/21031730d1ed41a68fd5a6cef21a2730, delay=907 ms 2024-12-06T01:45:45,581 DEBUG [StoreOpener-1588230740-1 {}] regionserver.StoreEngine(278): loaded file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/eff72b7a-3322-3fdb-88e1-7287cd98d0d4/e93bd390-0caa-6c00-f03a-52237a196617/data/hbase/meta/1588230740/ns/21031730d1ed41a68fd5a6cef21a2730 2024-12-06T01:45:45,581 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-06T01:45:45,581 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family rep_barrier of region 1588230740 2024-12-06T01:45:45,582 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-12-06T01:45:45,582 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T01:45:45,583 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-06T01:45:45,583 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family table of region 1588230740 2024-12-06T01:45:45,583 DEBUG [RS:2;00c764f7c3aa:36715 {}] zookeeper.ZKUtil(111): regionserver:36715-0x1006500a2f60003, quorum=127.0.0.1:61049, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/00c764f7c3aa,36715,1733449545505 2024-12-06T01:45:45,583 WARN [RS:2;00c764f7c3aa:36715 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-12-06T01:45:45,583 INFO [RS:2;00c764f7c3aa:36715 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-06T01:45:45,583 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-12-06T01:45:45,583 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T01:45:45,583 DEBUG [RS:2;00c764f7c3aa:36715 {}] regionserver.HRegionServer(1793): logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/eff72b7a-3322-3fdb-88e1-7287cd98d0d4/e93bd390-0caa-6c00-f03a-52237a196617/WALs/00c764f7c3aa,36715,1733449545505 2024-12-06T01:45:45,585 DEBUG [StoreFileOpener-1588230740-table-1 {}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/eff72b7a-3322-3fdb-88e1-7287cd98d0d4/e93bd390-0caa-6c00-f03a-52237a196617/data/hbase/meta/1588230740/table/219cd3ddf070484aaebb17561edcedaf, delay=910 ms 2024-12-06T01:45:45,586 DEBUG [StoreOpener-1588230740-1 {}] regionserver.StoreEngine(278): loaded file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/eff72b7a-3322-3fdb-88e1-7287cd98d0d4/e93bd390-0caa-6c00-f03a-52237a196617/data/hbase/meta/1588230740/table/219cd3ddf070484aaebb17561edcedaf 2024-12-06T01:45:45,586 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-06T01:45:45,587 DEBUG [RS_OPEN_META-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-12-06T01:45:45,587 DEBUG [RS_OPEN_META-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/eff72b7a-3322-3fdb-88e1-7287cd98d0d4/e93bd390-0caa-6c00-f03a-52237a196617/data/hbase/meta/1588230740 2024-12-06T01:45:45,588 DEBUG [RS_OPEN_META-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/eff72b7a-3322-3fdb-88e1-7287cd98d0d4/e93bd390-0caa-6c00-f03a-52237a196617/data/hbase/meta/1588230740 2024-12-06T01:45:45,588 DEBUG [RS_OPEN_META-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-12-06T01:45:45,588 DEBUG [RS_OPEN_META-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-12-06T01:45:45,589 DEBUG [RS_OPEN_META-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-12-06T01:45:45,590 DEBUG [RS_OPEN_META-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-12-06T01:45:45,590 INFO [RS:2;00c764f7c3aa:36715 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-12-06T01:45:45,590 INFO [RS_OPEN_META-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=15; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=65334863, jitterRate=-0.026434674859046936}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-12-06T01:45:45,591 DEBUG [RS_OPEN_META-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-12-06T01:45:45,593 DEBUG [RS_OPEN_META-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1733449545567Writing region info on filesystem at 1733449545567Initializing all the Stores at 1733449545567Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733449545567Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733449545568 (+1 ms)Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733449545568Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733449545568Cleaning up temporary data from old regions at 1733449545588 (+20 ms)Running coprocessor post-open hooks at 1733449545591 (+3 ms)Region opened successfully at 1733449545592 (+1 ms) 2024-12-06T01:45:45,593 INFO [RS_OPEN_META-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=9, masterSystemTime=1733449545538 2024-12-06T01:45:45,596 INFO [RS:2;00c764f7c3aa:36715 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-12-06T01:45:45,597 INFO [RS:2;00c764f7c3aa:36715 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-12-06T01:45:45,597 INFO [RS:2;00c764f7c3aa:36715 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-12-06T01:45:45,597 DEBUG [RS_OPEN_META-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-12-06T01:45:45,597 INFO [RS_OPEN_META-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_OPEN_META, pid=9}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-12-06T01:45:45,597 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=8 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=15, regionLocation=00c764f7c3aa,46757,1733449542231 2024-12-06T01:45:45,597 INFO [RS:2;00c764f7c3aa:36715 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-12-06T01:45:45,598 INFO [RS:2;00c764f7c3aa:36715 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-12-06T01:45:45,598 INFO [RS:2;00c764f7c3aa:36715 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-12-06T01:45:45,598 INFO [PEWorker-1 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 00c764f7c3aa,46757,1733449542231, state=OPEN 2024-12-06T01:45:45,598 DEBUG [RS:2;00c764f7c3aa:36715 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/00c764f7c3aa:0, corePoolSize=1, maxPoolSize=1 2024-12-06T01:45:45,598 DEBUG [RS:2;00c764f7c3aa:36715 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/00c764f7c3aa:0, corePoolSize=1, maxPoolSize=1 2024-12-06T01:45:45,598 DEBUG [RS:2;00c764f7c3aa:36715 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/00c764f7c3aa:0, corePoolSize=1, maxPoolSize=1 2024-12-06T01:45:45,598 DEBUG [RS:2;00c764f7c3aa:36715 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/00c764f7c3aa:0, corePoolSize=1, maxPoolSize=1 2024-12-06T01:45:45,598 DEBUG [RS:2;00c764f7c3aa:36715 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/00c764f7c3aa:0, corePoolSize=1, maxPoolSize=1 2024-12-06T01:45:45,598 DEBUG [RS:2;00c764f7c3aa:36715 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/00c764f7c3aa:0, corePoolSize=2, maxPoolSize=2 2024-12-06T01:45:45,598 DEBUG [RS:2;00c764f7c3aa:36715 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/00c764f7c3aa:0, corePoolSize=1, maxPoolSize=1 2024-12-06T01:45:45,598 DEBUG [RS:2;00c764f7c3aa:36715 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/00c764f7c3aa:0, corePoolSize=1, maxPoolSize=1 2024-12-06T01:45:45,599 DEBUG [RS:2;00c764f7c3aa:36715 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/00c764f7c3aa:0, corePoolSize=1, maxPoolSize=1 2024-12-06T01:45:45,599 DEBUG [RS:2;00c764f7c3aa:36715 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/00c764f7c3aa:0, corePoolSize=1, maxPoolSize=1 2024-12-06T01:45:45,599 DEBUG [RS:2;00c764f7c3aa:36715 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/00c764f7c3aa:0, corePoolSize=1, maxPoolSize=1 2024-12-06T01:45:45,599 DEBUG [RS:2;00c764f7c3aa:36715 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/00c764f7c3aa:0, corePoolSize=1, maxPoolSize=1 2024-12-06T01:45:45,599 DEBUG [RS:2;00c764f7c3aa:36715 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/00c764f7c3aa:0, corePoolSize=3, maxPoolSize=3 2024-12-06T01:45:45,599 DEBUG [RS:2;00c764f7c3aa:36715 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/00c764f7c3aa:0, corePoolSize=3, maxPoolSize=3 2024-12-06T01:45:45,599 INFO [RS:2;00c764f7c3aa:36715 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-12-06T01:45:45,599 INFO [RS:2;00c764f7c3aa:36715 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-12-06T01:45:45,600 INFO [RS:2;00c764f7c3aa:36715 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-06T01:45:45,600 INFO [RS:2;00c764f7c3aa:36715 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-12-06T01:45:45,600 INFO [RS:2;00c764f7c3aa:36715 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-12-06T01:45:45,600 INFO [RS:2;00c764f7c3aa:36715 {}] hbase.ChoreService(168): Chore ScheduledChore name=00c764f7c3aa,36715,1733449545505-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-12-06T01:45:45,600 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36715-0x1006500a2f60003, quorum=127.0.0.1:61049, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-06T01:45:45,600 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37657-0x1006500a2f60000, quorum=127.0.0.1:61049, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-06T01:45:45,600 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46757-0x1006500a2f60001, quorum=127.0.0.1:61049, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-06T01:45:45,600 DEBUG [PEWorker-1 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=9, ppid=8, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=00c764f7c3aa,46757,1733449542231 2024-12-06T01:45:45,600 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-06T01:45:45,600 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-06T01:45:45,601 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-06T01:45:45,602 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=9, resume processing ppid=8 2024-12-06T01:45:45,602 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=9, ppid=8, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=00c764f7c3aa,46757,1733449542231 in 214 msec 2024-12-06T01:45:45,605 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=8, resume processing ppid=7 2024-12-06T01:45:45,605 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=8, ppid=7, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 373 msec 2024-12-06T01:45:45,606 INFO [PEWorker-3 {}] procedure.ServerCrashProcedure(207): 00c764f7c3aa,45953,1733449542265 had 2 regions 2024-12-06T01:45:45,607 INFO [PEWorker-3 {}] procedure.ServerCrashProcedure(339): Splitting WALs pid=7, state=RUNNABLE:SERVER_CRASH_SPLIT_LOGS, hasLock=true; ServerCrashProcedure 00c764f7c3aa,45953,1733449542265, splitWal=true, meta=true, isMeta: false 2024-12-06T01:45:45,607 INFO [PEWorker-3 {}] master.SplitLogManager(171): file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/eff72b7a-3322-3fdb-88e1-7287cd98d0d4/e93bd390-0caa-6c00-f03a-52237a196617/WALs/00c764f7c3aa,45953,1733449542265-splitting dir is empty, no logs to split. 2024-12-06T01:45:45,607 INFO [PEWorker-3 {}] master.SplitWALManager(105): 00c764f7c3aa,45953,1733449542265 WAL count=0, meta=false 2024-12-06T01:45:45,609 INFO [PEWorker-3 {}] master.SplitLogManager(171): file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/eff72b7a-3322-3fdb-88e1-7287cd98d0d4/e93bd390-0caa-6c00-f03a-52237a196617/WALs/00c764f7c3aa,45953,1733449542265-splitting dir is empty, no logs to split. 2024-12-06T01:45:45,609 INFO [PEWorker-3 {}] master.SplitWALManager(105): 00c764f7c3aa,45953,1733449542265 WAL count=0, meta=false 2024-12-06T01:45:45,609 DEBUG [PEWorker-3 {}] procedure.ServerCrashProcedure(329): Check if 00c764f7c3aa,45953,1733449542265 WAL splitting is done? wals=0, meta=false 2024-12-06T01:45:45,610 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=10, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1733449543301, region=d6945b918e4d33856aa9730069614ec9, ASSIGN}] 2024-12-06T01:45:45,612 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=10, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1733449543301, region=d6945b918e4d33856aa9730069614ec9, ASSIGN 2024-12-06T01:45:45,612 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=10, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1733449543301, region=d6945b918e4d33856aa9730069614ec9, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-12-06T01:45:45,623 INFO [RS:2;00c764f7c3aa:36715 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-12-06T01:45:45,623 INFO [RS:2;00c764f7c3aa:36715 {}] hbase.ChoreService(168): Chore ScheduledChore name=00c764f7c3aa,36715,1733449545505-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-06T01:45:45,623 INFO [RS:2;00c764f7c3aa:36715 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-12-06T01:45:45,623 INFO [RS:2;00c764f7c3aa:36715 {}] regionserver.Replication(171): 00c764f7c3aa,36715,1733449545505 started 2024-12-06T01:45:45,640 INFO [RS:2;00c764f7c3aa:36715 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-12-06T01:45:45,641 INFO [RS:2;00c764f7c3aa:36715 {}] regionserver.HRegionServer(1482): Serving as 00c764f7c3aa,36715,1733449545505, RpcServer on 00c764f7c3aa/172.17.0.2:36715, sessionid=0x1006500a2f60003 2024-12-06T01:45:45,641 DEBUG [RS:2;00c764f7c3aa:36715 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-12-06T01:45:45,641 DEBUG [RS:2;00c764f7c3aa:36715 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 00c764f7c3aa,36715,1733449545505 2024-12-06T01:45:45,641 DEBUG [RS:2;00c764f7c3aa:36715 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '00c764f7c3aa,36715,1733449545505' 2024-12-06T01:45:45,641 DEBUG [RS:2;00c764f7c3aa:36715 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-12-06T01:45:45,642 DEBUG [RS:2;00c764f7c3aa:36715 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-12-06T01:45:45,642 DEBUG [RS:2;00c764f7c3aa:36715 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-12-06T01:45:45,642 DEBUG [RS:2;00c764f7c3aa:36715 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-12-06T01:45:45,642 DEBUG [RS:2;00c764f7c3aa:36715 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 00c764f7c3aa,36715,1733449545505 2024-12-06T01:45:45,642 DEBUG [RS:2;00c764f7c3aa:36715 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '00c764f7c3aa,36715,1733449545505' 2024-12-06T01:45:45,642 DEBUG [RS:2;00c764f7c3aa:36715 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-12-06T01:45:45,643 DEBUG [RS:2;00c764f7c3aa:36715 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-12-06T01:45:45,643 DEBUG [RS:2;00c764f7c3aa:36715 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-12-06T01:45:45,643 INFO [RS:2;00c764f7c3aa:36715 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-12-06T01:45:45,643 INFO [RS:2;00c764f7c3aa:36715 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-12-06T01:45:45,744 INFO [RS:2;00c764f7c3aa:36715 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-06T01:45:45,745 INFO [RS:2;00c764f7c3aa:36715 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=00c764f7c3aa%2C36715%2C1733449545505, suffix=, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/eff72b7a-3322-3fdb-88e1-7287cd98d0d4/e93bd390-0caa-6c00-f03a-52237a196617/WALs/00c764f7c3aa,36715,1733449545505, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/eff72b7a-3322-3fdb-88e1-7287cd98d0d4/e93bd390-0caa-6c00-f03a-52237a196617/oldWALs, maxLogs=55 2024-12-06T01:45:45,763 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=10 updating hbase:meta row=d6945b918e4d33856aa9730069614ec9, regionState=OPENING, regionLocation=00c764f7c3aa,46757,1733449542231 2024-12-06T01:45:45,765 INFO [RS:2;00c764f7c3aa:36715 {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/eff72b7a-3322-3fdb-88e1-7287cd98d0d4/e93bd390-0caa-6c00-f03a-52237a196617/WALs/00c764f7c3aa,36715,1733449545505/00c764f7c3aa%2C36715%2C1733449545505.1733449545746 2024-12-06T01:45:45,765 WARN [MiniHBaseClusterRegionServer-EventLoopGroup-8-3 {}] ipc.NettyRpcConnection$2(409): Exception encountered while connecting to the server 00c764f7c3aa:45953 org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 00c764f7c3aa/172.17.0.2:45953 Caused by: java.net.ConnectException: finishConnect(..) failed: Connection refused at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.newConnectException0(Errors.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.handleConnectErrno(Errors.java:131) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Socket.finishConnect(Socket.java:359) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.doFinishConnect(AbstractEpollChannel.java:715) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:692) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T01:45:45,767 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-8-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=00c764f7c3aa,45953,1733449542265, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=00c764f7c3aa,45953,1733449542265, seqNum=-1, error=java.net.ConnectException: Call to address=00c764f7c3aa:45953 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 00c764f7c3aa/172.17.0.2:45953 2024-12-06T01:45:45,767 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-8-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:meta,,1.1588230740, hostname=00c764f7c3aa,45953,1733449542265, seqNum=-1 is java.net.ConnectException: finishConnect(..) failed: Connection refused 2024-12-06T01:45:45,767 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-8-3 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:meta,,1.1588230740, hostname=00c764f7c3aa,45953,1733449542265, seqNum=-1 from cache 2024-12-06T01:45:45,767 DEBUG [RS:2;00c764f7c3aa:36715 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-12-06T01:45:45,768 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-8-3 {}] ipc.FailedServers(52): Added failed server with address 00c764f7c3aa:45953 to list caused by org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 00c764f7c3aa/172.17.0.2:45953 2024-12-06T01:45:45,877 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-06T01:45:45,877 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=00c764f7c3aa,46757,1733449542231, seqNum=-1] 2024-12-06T01:45:45,878 DEBUG [Async-Client-Retry-Timer-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-06T01:45:45,880 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-7-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34689, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-06T01:45:45,882 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-8-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=10, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1733449543301, region=d6945b918e4d33856aa9730069614ec9, ASSIGN because future has completed 2024-12-06T01:45:45,883 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=11, ppid=10, state=RUNNABLE, hasLock=false; OpenRegionProcedure d6945b918e4d33856aa9730069614ec9, server=00c764f7c3aa,46757,1733449542231}] 2024-12-06T01:45:46,037 INFO [RS_OPEN_REGION-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] handler.AssignRegionHandler(132): Open testBlockEvictionOnGracefulStop1733449543301,,1733449543305.d6945b918e4d33856aa9730069614ec9. 2024-12-06T01:45:46,038 DEBUG [RS_OPEN_REGION-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(7752): Opening region: {ENCODED => d6945b918e4d33856aa9730069614ec9, NAME => 'testBlockEvictionOnGracefulStop1733449543301,,1733449543305.d6945b918e4d33856aa9730069614ec9.', STARTKEY => '', ENDKEY => ''} 2024-12-06T01:45:46,039 DEBUG [RS_OPEN_REGION-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testBlockEvictionOnGracefulStop1733449543301 d6945b918e4d33856aa9730069614ec9 2024-12-06T01:45:46,039 DEBUG [RS_OPEN_REGION-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(898): Instantiated testBlockEvictionOnGracefulStop1733449543301,,1733449543305.d6945b918e4d33856aa9730069614ec9.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-06T01:45:46,039 DEBUG [RS_OPEN_REGION-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(7794): checking encryption for d6945b918e4d33856aa9730069614ec9 2024-12-06T01:45:46,039 DEBUG [RS_OPEN_REGION-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(7797): checking classloading for d6945b918e4d33856aa9730069614ec9 2024-12-06T01:45:46,040 INFO [StoreOpener-d6945b918e4d33856aa9730069614ec9-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family family of region d6945b918e4d33856aa9730069614ec9 2024-12-06T01:45:46,041 INFO [StoreOpener-d6945b918e4d33856aa9730069614ec9-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region d6945b918e4d33856aa9730069614ec9 columnFamilyName family 2024-12-06T01:45:46,041 DEBUG [StoreOpener-d6945b918e4d33856aa9730069614ec9-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T01:45:46,042 DEBUG [StoreFileOpener-d6945b918e4d33856aa9730069614ec9-family-1 {}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/eff72b7a-3322-3fdb-88e1-7287cd98d0d4/e93bd390-0caa-6c00-f03a-52237a196617/data/default/testBlockEvictionOnGracefulStop1733449543301/d6945b918e4d33856aa9730069614ec9/family/95052d11d88d4e2ca3db41260efa7f0e, delay=969 ms 2024-12-06T01:45:46,043 DEBUG [StoreOpener-d6945b918e4d33856aa9730069614ec9-1 {}] regionserver.StoreEngine(278): loaded file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/eff72b7a-3322-3fdb-88e1-7287cd98d0d4/e93bd390-0caa-6c00-f03a-52237a196617/data/default/testBlockEvictionOnGracefulStop1733449543301/d6945b918e4d33856aa9730069614ec9/family/95052d11d88d4e2ca3db41260efa7f0e 2024-12-06T01:45:46,044 INFO [StoreOpener-d6945b918e4d33856aa9730069614ec9-1 {}] regionserver.HStore(327): Store=d6945b918e4d33856aa9730069614ec9/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-06T01:45:46,044 DEBUG [RS_OPEN_REGION-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(1038): replaying wal for d6945b918e4d33856aa9730069614ec9 2024-12-06T01:45:46,045 DEBUG [RS_OPEN_REGION-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/eff72b7a-3322-3fdb-88e1-7287cd98d0d4/e93bd390-0caa-6c00-f03a-52237a196617/data/default/testBlockEvictionOnGracefulStop1733449543301/d6945b918e4d33856aa9730069614ec9 2024-12-06T01:45:46,045 DEBUG [RS_OPEN_REGION-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/eff72b7a-3322-3fdb-88e1-7287cd98d0d4/e93bd390-0caa-6c00-f03a-52237a196617/data/default/testBlockEvictionOnGracefulStop1733449543301/d6945b918e4d33856aa9730069614ec9 2024-12-06T01:45:46,045 DEBUG [RS_OPEN_REGION-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(1048): stopping wal replay for d6945b918e4d33856aa9730069614ec9 2024-12-06T01:45:46,046 DEBUG [RS_OPEN_REGION-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(1060): Cleaning up temporary data for d6945b918e4d33856aa9730069614ec9 2024-12-06T01:45:46,047 DEBUG [RS_OPEN_REGION-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(1093): writing seq id for d6945b918e4d33856aa9730069614ec9 2024-12-06T01:45:46,047 INFO [RS_OPEN_REGION-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(1114): Opened d6945b918e4d33856aa9730069614ec9; next sequenceid=10; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=75231821, jitterRate=0.12104149162769318}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-06T01:45:46,048 DEBUG [RS_OPEN_REGION-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(1122): Running coprocessor post-open hooks for d6945b918e4d33856aa9730069614ec9 2024-12-06T01:45:46,049 DEBUG [RS_OPEN_REGION-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(1006): Region open journal for d6945b918e4d33856aa9730069614ec9: Running coprocessor pre-open hook at 1733449546039Writing region info on filesystem at 1733449546039Initializing all the Stores at 1733449546040 (+1 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733449546040Cleaning up temporary data from old regions at 1733449546046 (+6 ms)Running coprocessor post-open hooks at 1733449546048 (+2 ms)Region opened successfully at 1733449546049 (+1 ms) 2024-12-06T01:45:46,050 INFO [RS_OPEN_REGION-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegionServer(2236): Post open deploy tasks for testBlockEvictionOnGracefulStop1733449543301,,1733449543305.d6945b918e4d33856aa9730069614ec9., pid=11, masterSystemTime=1733449546035 2024-12-06T01:45:46,053 DEBUG [RS_OPEN_REGION-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegionServer(2266): Finished post open deploy task for testBlockEvictionOnGracefulStop1733449543301,,1733449543305.d6945b918e4d33856aa9730069614ec9. 2024-12-06T01:45:46,053 INFO [RS_OPEN_REGION-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] handler.AssignRegionHandler(153): Opened testBlockEvictionOnGracefulStop1733449543301,,1733449543305.d6945b918e4d33856aa9730069614ec9. 2024-12-06T01:45:46,053 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=10 updating hbase:meta row=d6945b918e4d33856aa9730069614ec9, regionState=OPEN, openSeqNum=10, regionLocation=00c764f7c3aa,46757,1733449542231 2024-12-06T01:45:46,056 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-8-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=11, ppid=10, state=RUNNABLE, hasLock=false; OpenRegionProcedure d6945b918e4d33856aa9730069614ec9, server=00c764f7c3aa,46757,1733449542231 because future has completed 2024-12-06T01:45:46,059 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=11, resume processing ppid=10 2024-12-06T01:45:46,059 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=11, ppid=10, state=SUCCESS, hasLock=false; OpenRegionProcedure d6945b918e4d33856aa9730069614ec9, server=00c764f7c3aa,46757,1733449542231 in 174 msec 2024-12-06T01:45:46,061 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=10, resume processing ppid=7 2024-12-06T01:45:46,061 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=10, ppid=7, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1733449543301, region=d6945b918e4d33856aa9730069614ec9, ASSIGN in 449 msec 2024-12-06T01:45:46,061 INFO [PEWorker-1 {}] procedure.ServerCrashProcedure(291): removed crashed server 00c764f7c3aa,45953,1733449542265 after splitting done 2024-12-06T01:45:46,063 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=7, state=SUCCESS, hasLock=false; ServerCrashProcedure 00c764f7c3aa,45953,1733449542265, splitWal=true, meta=true in 844 msec 2024-12-06T01:45:46,141 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster 2024-12-06T01:45:46,141 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-12-06T01:45:46,141 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79) at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611) at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.io.hfile.TestBlockEvictionOnRegionMovement.tearDown(TestBlockEvictionOnRegionMovement.java:174) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-06T01:45:46,141 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-06T01:45:46,141 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-06T01:45:46,141 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-06T01:45:46,141 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2024-12-06T01:45:46,142 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=113318242, stopped=false 2024-12-06T01:45:46,142 INFO [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=00c764f7c3aa,37657,1733449542177 2024-12-06T01:45:46,143 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36715-0x1006500a2f60003, quorum=127.0.0.1:61049, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-12-06T01:45:46,143 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46757-0x1006500a2f60001, quorum=127.0.0.1:61049, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-12-06T01:45:46,144 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37657-0x1006500a2f60000, quorum=127.0.0.1:61049, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-12-06T01:45:46,144 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36715-0x1006500a2f60003, quorum=127.0.0.1:61049, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-06T01:45:46,144 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46757-0x1006500a2f60001, quorum=127.0.0.1:61049, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-06T01:45:46,144 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37657-0x1006500a2f60000, quorum=127.0.0.1:61049, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-06T01:45:46,144 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-06T01:45:46,144 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-12-06T01:45:46,144 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277) at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265) at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416) at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.io.hfile.TestBlockEvictionOnRegionMovement.tearDown(TestBlockEvictionOnRegionMovement.java:174) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-06T01:45:46,144 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-06T01:45:46,144 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:46757-0x1006500a2f60001, quorum=127.0.0.1:61049, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-06T01:45:46,145 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:37657-0x1006500a2f60000, quorum=127.0.0.1:61049, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-06T01:45:46,145 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:36715-0x1006500a2f60003, quorum=127.0.0.1:61049, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-06T01:45:46,145 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '00c764f7c3aa,46757,1733449542231' ***** 2024-12-06T01:45:46,145 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-12-06T01:45:46,145 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '00c764f7c3aa,36715,1733449545505' ***** 2024-12-06T01:45:46,146 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-12-06T01:45:46,146 INFO [RS:2;00c764f7c3aa:36715 {}] bucket.BucketCache(1738): Disabling cache 2024-12-06T01:45:46,146 INFO [RS:2;00c764f7c3aa:36715 {}] bucket.BucketCache(1762): Shutdown bucket cache: IO persistent=true; path to write=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/21f224f3-974a-889b-42d4-cdd5a08d3a54/bucket.persistence 2024-12-06T01:45:46,146 INFO [RS:0;00c764f7c3aa:46757 {}] bucket.BucketCache(1738): Disabling cache 2024-12-06T01:45:46,146 INFO [Time-limited test-BucketCacheWriter-0 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-0 exiting, cacheEnabled=false 2024-12-06T01:45:46,146 INFO [Time-limited test-BucketCacheWriter-1 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-1 exiting, cacheEnabled=false 2024-12-06T01:45:46,146 INFO [Time-limited test-BucketCacheWriter-2 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-2 exiting, cacheEnabled=false 2024-12-06T01:45:46,146 INFO [RS:0;00c764f7c3aa:46757 {}] bucket.BucketCache(1762): Shutdown bucket cache: IO persistent=true; path to write=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/21f224f3-974a-889b-42d4-cdd5a08d3a54/bucket.persistence 2024-12-06T01:45:46,146 INFO [Time-limited test-BucketCacheWriter-0 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-0 exiting, cacheEnabled=false 2024-12-06T01:45:46,146 INFO [Time-limited test-BucketCacheWriter-2 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-2 exiting, cacheEnabled=false 2024-12-06T01:45:46,147 INFO [Time-limited test-BucketCacheWriter-1 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-1 exiting, cacheEnabled=false 2024-12-06T01:45:46,147 INFO [RS:0;00c764f7c3aa:46757 {}] bucket.BucketCache(1768): Shutting down cache persister thread. 2024-12-06T01:45:46,147 INFO [RS:2;00c764f7c3aa:36715 {}] bucket.BucketCache(1768): Shutting down cache persister thread. 2024-12-06T01:45:46,147 WARN [bucket-cache-persister {}] bucket.BucketCachePersister(64): Interrupting BucketCachePersister thread. java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCachePersister.run(BucketCachePersister.java:45) ~[classes/:?] 2024-12-06T01:45:46,147 WARN [bucket-cache-persister {}] bucket.BucketCachePersister(64): Interrupting BucketCachePersister thread. java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCachePersister.run(BucketCachePersister.java:45) ~[classes/:?] 2024-12-06T01:45:46,151 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-12-06T01:45:46,152 INFO [regionserver/00c764f7c3aa:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-12-06T01:45:46,157 DEBUG [RS:0;00c764f7c3aa:46757 {}] bucket.BucketCache(1487): Thread RS:0;00c764f7c3aa:46757 started persisting bucket cache to file 2024-12-06T01:45:46,157 DEBUG [RS:2;00c764f7c3aa:36715 {}] bucket.BucketCache(1487): Thread RS:2;00c764f7c3aa:36715 started persisting bucket cache to file 2024-12-06T01:45:46,157 DEBUG [RS:0;00c764f7c3aa:46757 {}] bucket.BucketCache(1494): Persist in new chunked persistence format. 2024-12-06T01:45:46,157 DEBUG [RS:2;00c764f7c3aa:36715 {}] bucket.BucketCache(1494): Persist in new chunked persistence format. 2024-12-06T01:45:46,157 DEBUG [RS:0;00c764f7c3aa:46757 {}] bucket.BucketCache(1673): persistToFile: before persisting backing map size: 0, fullycachedFiles size: 0, chunkSize: 10000 2024-12-06T01:45:46,157 DEBUG [RS:2;00c764f7c3aa:36715 {}] bucket.BucketCache(1673): persistToFile: before persisting backing map size: 1, fullycachedFiles size: 0, chunkSize: 10000 2024-12-06T01:45:46,161 DEBUG [RS:0;00c764f7c3aa:46757 {}] bucket.BucketCache(1680): persistToFile: after persisting backing map size: 0, fullycachedFiles size: 0 2024-12-06T01:45:46,161 DEBUG [RS:0;00c764f7c3aa:46757 {}] bucket.BucketCache(1498): PersistToFile: after persisting backing map size: 0, fullycachedFiles size: 0, file name: bucket.persistence1733449546157 2024-12-06T01:45:46,161 DEBUG [RS:2;00c764f7c3aa:36715 {}] bucket.BucketCache(1680): persistToFile: after persisting backing map size: 1, fullycachedFiles size: 0 2024-12-06T01:45:46,161 DEBUG [RS:2;00c764f7c3aa:36715 {}] bucket.BucketCache(1498): PersistToFile: after persisting backing map size: 1, fullycachedFiles size: 0, file name: bucket.persistence1733449546157 2024-12-06T01:45:46,161 DEBUG [RS:0;00c764f7c3aa:46757 {}] bucket.BucketCache(1509): Thread RS:0;00c764f7c3aa:46757 finished persisting bucket cache to file, renaming 2024-12-06T01:45:46,161 DEBUG [RS:2;00c764f7c3aa:36715 {}] bucket.BucketCache(1509): Thread RS:2;00c764f7c3aa:36715 finished persisting bucket cache to file, renaming 2024-12-06T01:45:46,161 WARN [RS:2;00c764f7c3aa:36715 {}] bucket.BucketCache(1512): Failed to commit cache persistent file. We might lose cached blocks if RS crashes/restarts before we successfully checkpoint again. 2024-12-06T01:45:46,161 INFO [RS:0;00c764f7c3aa:46757 {}] regionserver.HeapMemoryManager(220): Stopping 2024-12-06T01:45:46,161 INFO [RS:2;00c764f7c3aa:36715 {}] regionserver.HeapMemoryManager(220): Stopping 2024-12-06T01:45:46,161 INFO [RS:2;00c764f7c3aa:36715 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-12-06T01:45:46,161 INFO [RS:0;00c764f7c3aa:46757 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-12-06T01:45:46,161 INFO [RS:0;00c764f7c3aa:46757 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-12-06T01:45:46,161 INFO [RS:2;00c764f7c3aa:36715 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-12-06T01:45:46,161 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-12-06T01:45:46,162 INFO [RS:2;00c764f7c3aa:36715 {}] regionserver.HRegionServer(959): stopping server 00c764f7c3aa,36715,1733449545505 2024-12-06T01:45:46,162 INFO [RS:0;00c764f7c3aa:46757 {}] regionserver.HRegionServer(3091): Received CLOSE for d6945b918e4d33856aa9730069614ec9 2024-12-06T01:45:46,162 INFO [RS:2;00c764f7c3aa:36715 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-12-06T01:45:46,162 INFO [RS:2;00c764f7c3aa:36715 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:2;00c764f7c3aa:36715. 2024-12-06T01:45:46,162 DEBUG [RS:2;00c764f7c3aa:36715 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-06T01:45:46,162 DEBUG [RS:2;00c764f7c3aa:36715 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-06T01:45:46,162 INFO [RS:0;00c764f7c3aa:46757 {}] regionserver.HRegionServer(959): stopping server 00c764f7c3aa,46757,1733449542231 2024-12-06T01:45:46,162 INFO [RS:0;00c764f7c3aa:46757 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-12-06T01:45:46,162 INFO [RS:2;00c764f7c3aa:36715 {}] regionserver.HRegionServer(976): stopping server 00c764f7c3aa,36715,1733449545505; all regions closed. 2024-12-06T01:45:46,162 INFO [RS:0;00c764f7c3aa:46757 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;00c764f7c3aa:46757. 2024-12-06T01:45:46,162 DEBUG [RS:0;00c764f7c3aa:46757 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-06T01:45:46,162 INFO [RS_CLOSE_REGION-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1662): Closing region testBlockEvictionOnGracefulStop1733449543301,,1733449543305.d6945b918e4d33856aa9730069614ec9. during a graceful stop, and cache persistence is on, so setting evict on close to false. 2024-12-06T01:45:46,162 DEBUG [RS:0;00c764f7c3aa:46757 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-06T01:45:46,162 DEBUG [RS_CLOSE_REGION-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing d6945b918e4d33856aa9730069614ec9, disabling compactions & flushes 2024-12-06T01:45:46,162 INFO [RS:0;00c764f7c3aa:46757 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-12-06T01:45:46,162 INFO [RS:0;00c764f7c3aa:46757 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-12-06T01:45:46,162 INFO [RS:0;00c764f7c3aa:46757 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-12-06T01:45:46,162 INFO [RS_CLOSE_REGION-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testBlockEvictionOnGracefulStop1733449543301,,1733449543305.d6945b918e4d33856aa9730069614ec9. 2024-12-06T01:45:46,163 INFO [RS:0;00c764f7c3aa:46757 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-12-06T01:45:46,163 DEBUG [RS_CLOSE_REGION-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testBlockEvictionOnGracefulStop1733449543301,,1733449543305.d6945b918e4d33856aa9730069614ec9. 2024-12-06T01:45:46,163 DEBUG [RS_CLOSE_REGION-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testBlockEvictionOnGracefulStop1733449543301,,1733449543305.d6945b918e4d33856aa9730069614ec9. after waiting 0 ms 2024-12-06T01:45:46,163 DEBUG [RS_CLOSE_REGION-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testBlockEvictionOnGracefulStop1733449543301,,1733449543305.d6945b918e4d33856aa9730069614ec9. 2024-12-06T01:45:46,163 INFO [RS:0;00c764f7c3aa:46757 {}] regionserver.HRegionServer(1321): Waiting on 2 regions to close 2024-12-06T01:45:46,163 DEBUG [RS:0;00c764f7c3aa:46757 {}] regionserver.HRegionServer(1325): Online Regions={1588230740=hbase:meta,,1.1588230740, d6945b918e4d33856aa9730069614ec9=testBlockEvictionOnGracefulStop1733449543301,,1733449543305.d6945b918e4d33856aa9730069614ec9.} 2024-12-06T01:45:46,163 DEBUG [RS:0;00c764f7c3aa:46757 {}] regionserver.HRegionServer(1351): Waiting on 1588230740, d6945b918e4d33856aa9730069614ec9 2024-12-06T01:45:46,163 INFO [RS_CLOSE_META-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1662): Closing region hbase:meta,,1.1588230740 during a graceful stop, and cache persistence is on, so setting evict on close to false. 2024-12-06T01:45:46,163 DEBUG [RS_CLOSE_META-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-12-06T01:45:46,163 INFO [RS_CLOSE_META-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-12-06T01:45:46,163 DEBUG [RS_CLOSE_META-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-12-06T01:45:46,164 DEBUG [RS_CLOSE_META-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-12-06T01:45:46,164 DEBUG [RS_CLOSE_META-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-12-06T01:45:46,164 DEBUG [StoreFileCloser-d6945b918e4d33856aa9730069614ec9-family-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/eff72b7a-3322-3fdb-88e1-7287cd98d0d4/e93bd390-0caa-6c00-f03a-52237a196617/data/default/testBlockEvictionOnGracefulStop1733449543301/d6945b918e4d33856aa9730069614ec9/family/95052d11d88d4e2ca3db41260efa7f0e 2024-12-06T01:45:46,164 INFO [RS_CLOSE_META-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=1.21 KB heapSize=2.80 KB 2024-12-06T01:45:46,165 DEBUG [RS:2;00c764f7c3aa:36715 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/eff72b7a-3322-3fdb-88e1-7287cd98d0d4/e93bd390-0caa-6c00-f03a-52237a196617/oldWALs 2024-12-06T01:45:46,165 INFO [RS:2;00c764f7c3aa:36715 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 00c764f7c3aa%2C36715%2C1733449545505:(num 1733449545746) 2024-12-06T01:45:46,165 DEBUG [RS:2;00c764f7c3aa:36715 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-06T01:45:46,165 INFO [RS:2;00c764f7c3aa:36715 {}] regionserver.LeaseManager(133): Closed leases 2024-12-06T01:45:46,165 INFO [RS:2;00c764f7c3aa:36715 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-12-06T01:45:46,165 INFO [RS:2;00c764f7c3aa:36715 {}] hbase.ChoreService(370): Chore service for: regionserver/00c764f7c3aa:0 had [ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-12-06T01:45:46,165 INFO [RS:2;00c764f7c3aa:36715 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-12-06T01:45:46,165 INFO [RS:2;00c764f7c3aa:36715 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-12-06T01:45:46,165 INFO [RS:2;00c764f7c3aa:36715 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-12-06T01:45:46,165 INFO [RS:2;00c764f7c3aa:36715 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-12-06T01:45:46,165 INFO [RS:2;00c764f7c3aa:36715 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:36715 2024-12-06T01:45:46,166 INFO [regionserver/00c764f7c3aa:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-12-06T01:45:46,168 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37657-0x1006500a2f60000, quorum=127.0.0.1:61049, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-12-06T01:45:46,168 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36715-0x1006500a2f60003, quorum=127.0.0.1:61049, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/00c764f7c3aa,36715,1733449545505 2024-12-06T01:45:46,168 INFO [RS:2;00c764f7c3aa:36715 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-12-06T01:45:46,169 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [00c764f7c3aa,36715,1733449545505] 2024-12-06T01:45:46,169 DEBUG [RS_CLOSE_REGION-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/eff72b7a-3322-3fdb-88e1-7287cd98d0d4/e93bd390-0caa-6c00-f03a-52237a196617/data/default/testBlockEvictionOnGracefulStop1733449543301/d6945b918e4d33856aa9730069614ec9/recovered.edits/12.seqid, newMaxSeqId=12, maxSeqId=9 2024-12-06T01:45:46,170 INFO [RS_CLOSE_REGION-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testBlockEvictionOnGracefulStop1733449543301,,1733449543305.d6945b918e4d33856aa9730069614ec9. 2024-12-06T01:45:46,170 DEBUG [RS_CLOSE_REGION-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for d6945b918e4d33856aa9730069614ec9: Waiting for close lock at 1733449546162Running coprocessor pre-close hooks at 1733449546162Disabling compacts and flushes for region at 1733449546162Disabling writes for close at 1733449546163 (+1 ms)Writing region close event to WAL at 1733449546164 (+1 ms)Running coprocessor post-close hooks at 1733449546170 (+6 ms)Closed at 1733449546170 2024-12-06T01:45:46,170 DEBUG [RS_CLOSE_REGION-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testBlockEvictionOnGracefulStop1733449543301,,1733449543305.d6945b918e4d33856aa9730069614ec9. 2024-12-06T01:45:46,171 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/00c764f7c3aa,36715,1733449545505 already deleted, retry=false 2024-12-06T01:45:46,171 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; 00c764f7c3aa,36715,1733449545505 expired; onlineServers=1 2024-12-06T01:45:46,176 DEBUG [RS_CLOSE_META-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 36c30ac300ad44ac84ede567f51becd4 2024-12-06T01:45:46,191 DEBUG [RS_CLOSE_META-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/eff72b7a-3322-3fdb-88e1-7287cd98d0d4/e93bd390-0caa-6c00-f03a-52237a196617/data/hbase/meta/1588230740/.tmp/info/36c30ac300ad44ac84ede567f51becd4 is 209, key is testBlockEvictionOnGracefulStop1733449543301,,1733449543305.d6945b918e4d33856aa9730069614ec9./info:regioninfo/1733449546053/Put/seqid=0 2024-12-06T01:45:46,191 INFO [RS_CLOSE_META-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.21 KB at sequenceid=19 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/eff72b7a-3322-3fdb-88e1-7287cd98d0d4/e93bd390-0caa-6c00-f03a-52237a196617/data/hbase/meta/1588230740/.tmp/info/36c30ac300ad44ac84ede567f51becd4 2024-12-06T01:45:46,194 DEBUG [RS_CLOSE_META-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/eff72b7a-3322-3fdb-88e1-7287cd98d0d4/e93bd390-0caa-6c00-f03a-52237a196617/data/hbase/meta/1588230740/.tmp/info/36c30ac300ad44ac84ede567f51becd4 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/eff72b7a-3322-3fdb-88e1-7287cd98d0d4/e93bd390-0caa-6c00-f03a-52237a196617/data/hbase/meta/1588230740/info/36c30ac300ad44ac84ede567f51becd4 2024-12-06T01:45:46,195 DEBUG [RS_CLOSE_META-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_CLOSE_META}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/eff72b7a-3322-3fdb-88e1-7287cd98d0d4/e93bd390-0caa-6c00-f03a-52237a196617/data/hbase/meta/1588230740/info/36c30ac300ad44ac84ede567f51becd4, delay=971 ms 2024-12-06T01:45:46,196 INFO [RS_CLOSE_META-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/eff72b7a-3322-3fdb-88e1-7287cd98d0d4/e93bd390-0caa-6c00-f03a-52237a196617/data/hbase/meta/1588230740/info/36c30ac300ad44ac84ede567f51becd4, entries=8, sequenceid=19, filesize=6.6 K 2024-12-06T01:45:46,197 INFO [RS_CLOSE_META-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~1.21 KB/1238, heapSize ~2.03 KB/2080, currentSize=0 B/0 for 1588230740 in 33ms, sequenceid=19, compaction requested=false 2024-12-06T01:45:46,198 DEBUG [StoreFileCloser-1588230740-info-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/eff72b7a-3322-3fdb-88e1-7287cd98d0d4/e93bd390-0caa-6c00-f03a-52237a196617/data/hbase/meta/1588230740/info/069acf6dc56e40a29632095fab5c92df 2024-12-06T01:45:46,199 DEBUG [StoreFileCloser-1588230740-info-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/eff72b7a-3322-3fdb-88e1-7287cd98d0d4/e93bd390-0caa-6c00-f03a-52237a196617/data/hbase/meta/1588230740/info/36c30ac300ad44ac84ede567f51becd4 2024-12-06T01:45:46,199 DEBUG [StoreFileCloser-1588230740-ns-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/eff72b7a-3322-3fdb-88e1-7287cd98d0d4/e93bd390-0caa-6c00-f03a-52237a196617/data/hbase/meta/1588230740/ns/21031730d1ed41a68fd5a6cef21a2730 2024-12-06T01:45:46,200 DEBUG [StoreFileCloser-1588230740-table-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/eff72b7a-3322-3fdb-88e1-7287cd98d0d4/e93bd390-0caa-6c00-f03a-52237a196617/data/hbase/meta/1588230740/table/219cd3ddf070484aaebb17561edcedaf 2024-12-06T01:45:46,204 INFO [regionserver/00c764f7c3aa:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-12-06T01:45:46,205 DEBUG [RS_CLOSE_META-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/eff72b7a-3322-3fdb-88e1-7287cd98d0d4/e93bd390-0caa-6c00-f03a-52237a196617/data/hbase/meta/1588230740/recovered.edits/22.seqid, newMaxSeqId=22, maxSeqId=14 2024-12-06T01:45:46,205 DEBUG [RS_CLOSE_META-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-12-06T01:45:46,205 INFO [RS_CLOSE_META-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-12-06T01:45:46,205 DEBUG [RS_CLOSE_META-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1733449546163Running coprocessor pre-close hooks at 1733449546163Disabling compacts and flushes for region at 1733449546163Disabling writes for close at 1733449546164 (+1 ms)Obtaining lock to block concurrent updates at 1733449546164Preparing flush snapshotting stores in 1588230740 at 1733449546164Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=1238, getHeapSize=2800, getOffHeapSize=0, getCellsCount=8 at 1733449546165 (+1 ms)Flushing stores of hbase:meta,,1.1588230740 at 1733449546165Flushing 1588230740/info: creating writer at 1733449546165Flushing 1588230740/info: appending metadata at 1733449546190 (+25 ms)Flushing 1588230740/info: closing flushed file at 1733449546190Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@6e606eb0: reopening flushed file at 1733449546194 (+4 ms)Finished flush of dataSize ~1.21 KB/1238, heapSize ~2.03 KB/2080, currentSize=0 B/0 for 1588230740 in 33ms, sequenceid=19, compaction requested=false at 1733449546197 (+3 ms)Writing region close event to WAL at 1733449546200 (+3 ms)Running coprocessor post-close hooks at 1733449546205 (+5 ms)Closed at 1733449546205 2024-12-06T01:45:46,206 DEBUG [RS_CLOSE_META-regionserver/00c764f7c3aa:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-12-06T01:45:46,270 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36715-0x1006500a2f60003, quorum=127.0.0.1:61049, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-06T01:45:46,270 INFO [RS:2;00c764f7c3aa:36715 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-12-06T01:45:46,270 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36715-0x1006500a2f60003, quorum=127.0.0.1:61049, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-06T01:45:46,270 INFO [RS:2;00c764f7c3aa:36715 {}] regionserver.HRegionServer(1031): Exiting; stopping=00c764f7c3aa,36715,1733449545505; zookeeper connection closed. 2024-12-06T01:45:46,271 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@5a8abf2b {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@5a8abf2b 2024-12-06T01:45:46,363 INFO [RS:0;00c764f7c3aa:46757 {}] regionserver.HRegionServer(976): stopping server 00c764f7c3aa,46757,1733449542231; all regions closed. 2024-12-06T01:45:46,365 DEBUG [RS:0;00c764f7c3aa:46757 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/eff72b7a-3322-3fdb-88e1-7287cd98d0d4/e93bd390-0caa-6c00-f03a-52237a196617/oldWALs 2024-12-06T01:45:46,365 INFO [RS:0;00c764f7c3aa:46757 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 00c764f7c3aa%2C46757%2C1733449542231.meta:.meta(num 1733449545545) 2024-12-06T01:45:46,367 DEBUG [RS:0;00c764f7c3aa:46757 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/eff72b7a-3322-3fdb-88e1-7287cd98d0d4/e93bd390-0caa-6c00-f03a-52237a196617/oldWALs 2024-12-06T01:45:46,367 INFO [RS:0;00c764f7c3aa:46757 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 00c764f7c3aa%2C46757%2C1733449542231:(num 1733449542808) 2024-12-06T01:45:46,367 DEBUG [RS:0;00c764f7c3aa:46757 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-06T01:45:46,367 INFO [RS:0;00c764f7c3aa:46757 {}] regionserver.LeaseManager(133): Closed leases 2024-12-06T01:45:46,367 INFO [RS:0;00c764f7c3aa:46757 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-12-06T01:45:46,368 INFO [RS:0;00c764f7c3aa:46757 {}] hbase.ChoreService(370): Chore service for: regionserver/00c764f7c3aa:0 had [ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-12-06T01:45:46,368 INFO [RS:0;00c764f7c3aa:46757 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-12-06T01:45:46,368 INFO [regionserver/00c764f7c3aa:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-12-06T01:45:46,368 INFO [RS:0;00c764f7c3aa:46757 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:46757 2024-12-06T01:45:46,371 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46757-0x1006500a2f60001, quorum=127.0.0.1:61049, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/00c764f7c3aa,46757,1733449542231 2024-12-06T01:45:46,371 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37657-0x1006500a2f60000, quorum=127.0.0.1:61049, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-12-06T01:45:46,371 INFO [RS:0;00c764f7c3aa:46757 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-12-06T01:45:46,372 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [00c764f7c3aa,46757,1733449542231] 2024-12-06T01:45:46,374 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/00c764f7c3aa,46757,1733449542231 already deleted, retry=false 2024-12-06T01:45:46,374 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; 00c764f7c3aa,46757,1733449542231 expired; onlineServers=0 2024-12-06T01:45:46,374 INFO [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master '00c764f7c3aa,37657,1733449542177' ***** 2024-12-06T01:45:46,374 INFO [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0 2024-12-06T01:45:46,374 INFO [M:0;00c764f7c3aa:37657 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-12-06T01:45:46,374 INFO [M:0;00c764f7c3aa:37657 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-12-06T01:45:46,374 DEBUG [M:0;00c764f7c3aa:37657 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2024-12-06T01:45:46,374 DEBUG [M:0;00c764f7c3aa:37657 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2024-12-06T01:45:46,374 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2024-12-06T01:45:46,374 DEBUG [master/00c764f7c3aa:0:becomeActiveMaster-HFileCleaner.large.0-1733449542511 {}] cleaner.HFileCleaner(306): Exit Thread[master/00c764f7c3aa:0:becomeActiveMaster-HFileCleaner.large.0-1733449542511,5,FailOnTimeoutGroup] 2024-12-06T01:45:46,374 DEBUG [master/00c764f7c3aa:0:becomeActiveMaster-HFileCleaner.small.0-1733449542511 {}] cleaner.HFileCleaner(306): Exit Thread[master/00c764f7c3aa:0:becomeActiveMaster-HFileCleaner.small.0-1733449542511,5,FailOnTimeoutGroup] 2024-12-06T01:45:46,375 INFO [M:0;00c764f7c3aa:37657 {}] hbase.ChoreService(370): Chore service for: master/00c764f7c3aa:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown 2024-12-06T01:45:46,375 INFO [M:0;00c764f7c3aa:37657 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-12-06T01:45:46,375 DEBUG [M:0;00c764f7c3aa:37657 {}] master.HMaster(1795): Stopping service threads 2024-12-06T01:45:46,375 INFO [M:0;00c764f7c3aa:37657 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2024-12-06T01:45:46,375 INFO [M:0;00c764f7c3aa:37657 {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-06T01:45:46,375 INFO [M:0;00c764f7c3aa:37657 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-12-06T01:45:46,376 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2024-12-06T01:45:46,376 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37657-0x1006500a2f60000, quorum=127.0.0.1:61049, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-12-06T01:45:46,377 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37657-0x1006500a2f60000, quorum=127.0.0.1:61049, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-06T01:45:46,377 DEBUG [M:0;00c764f7c3aa:37657 {}] zookeeper.ZKUtil(347): master:37657-0x1006500a2f60000, quorum=127.0.0.1:61049, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error) 2024-12-06T01:45:46,377 WARN [M:0;00c764f7c3aa:37657 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null 2024-12-06T01:45:46,377 INFO [M:0;00c764f7c3aa:37657 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/eff72b7a-3322-3fdb-88e1-7287cd98d0d4/e93bd390-0caa-6c00-f03a-52237a196617/.lastflushedseqids 2024-12-06T01:45:46,384 INFO [M:0;00c764f7c3aa:37657 {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-12-06T01:45:46,384 INFO [M:0;00c764f7c3aa:37657 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2024-12-06T01:45:46,385 DEBUG [M:0;00c764f7c3aa:37657 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-12-06T01:45:46,385 INFO [M:0;00c764f7c3aa:37657 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-06T01:45:46,385 DEBUG [M:0;00c764f7c3aa:37657 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-06T01:45:46,385 DEBUG [M:0;00c764f7c3aa:37657 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-12-06T01:45:46,385 DEBUG [M:0;00c764f7c3aa:37657 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-06T01:45:46,385 INFO [M:0;00c764f7c3aa:37657 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=39.50 KB heapSize=49.91 KB 2024-12-06T01:45:46,394 DEBUG [M:0;00c764f7c3aa:37657 {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 7d7ceb273ffb42128774953dce9fbb11 2024-12-06T01:45:46,409 DEBUG [M:0;00c764f7c3aa:37657 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/eff72b7a-3322-3fdb-88e1-7287cd98d0d4/e93bd390-0caa-6c00-f03a-52237a196617/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/7d7ceb273ffb42128774953dce9fbb11 is 82, key is hbase:meta,,1/info:regioninfo/1733449545597/Put/seqid=0 2024-12-06T01:45:46,410 INFO [M:0;00c764f7c3aa:37657 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1008 B at sequenceid=102 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/eff72b7a-3322-3fdb-88e1-7287cd98d0d4/e93bd390-0caa-6c00-f03a-52237a196617/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/7d7ceb273ffb42128774953dce9fbb11 2024-12-06T01:45:46,419 DEBUG [M:0;00c764f7c3aa:37657 {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 4b718bb9cc7d41fbac6d43fb66a6909e 2024-12-06T01:45:46,434 DEBUG [M:0;00c764f7c3aa:37657 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/eff72b7a-3322-3fdb-88e1-7287cd98d0d4/e93bd390-0caa-6c00-f03a-52237a196617/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/4b718bb9cc7d41fbac6d43fb66a6909e is 782, key is \x00\x00\x00\x00\x00\x00\x00\x04/proc:d/1733449543684/Put/seqid=0 2024-12-06T01:45:46,435 INFO [M:0;00c764f7c3aa:37657 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.27 KB at sequenceid=102 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/eff72b7a-3322-3fdb-88e1-7287cd98d0d4/e93bd390-0caa-6c00-f03a-52237a196617/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/4b718bb9cc7d41fbac6d43fb66a6909e 2024-12-06T01:45:46,443 DEBUG [M:0;00c764f7c3aa:37657 {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on ab53492d199048588c48c872c891d5b7 2024-12-06T01:45:46,457 DEBUG [M:0;00c764f7c3aa:37657 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/eff72b7a-3322-3fdb-88e1-7287cd98d0d4/e93bd390-0caa-6c00-f03a-52237a196617/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/ab53492d199048588c48c872c891d5b7 is 69, key is 00c764f7c3aa,36715,1733449545505/rs:state/1733449545575/Put/seqid=0 2024-12-06T01:45:46,458 INFO [M:0;00c764f7c3aa:37657 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=249 B at sequenceid=102 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/eff72b7a-3322-3fdb-88e1-7287cd98d0d4/e93bd390-0caa-6c00-f03a-52237a196617/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/ab53492d199048588c48c872c891d5b7 2024-12-06T01:45:46,459 INFO [M:0;00c764f7c3aa:37657 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for ab53492d199048588c48c872c891d5b7 2024-12-06T01:45:46,460 DEBUG [M:0;00c764f7c3aa:37657 {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/eff72b7a-3322-3fdb-88e1-7287cd98d0d4/e93bd390-0caa-6c00-f03a-52237a196617/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/7d7ceb273ffb42128774953dce9fbb11 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/eff72b7a-3322-3fdb-88e1-7287cd98d0d4/e93bd390-0caa-6c00-f03a-52237a196617/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/7d7ceb273ffb42128774953dce9fbb11 2024-12-06T01:45:46,461 INFO [M:0;00c764f7c3aa:37657 {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/eff72b7a-3322-3fdb-88e1-7287cd98d0d4/e93bd390-0caa-6c00-f03a-52237a196617/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/7d7ceb273ffb42128774953dce9fbb11, entries=14, sequenceid=102, filesize=5.9 K 2024-12-06T01:45:46,462 DEBUG [M:0;00c764f7c3aa:37657 {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/eff72b7a-3322-3fdb-88e1-7287cd98d0d4/e93bd390-0caa-6c00-f03a-52237a196617/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/4b718bb9cc7d41fbac6d43fb66a6909e as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/eff72b7a-3322-3fdb-88e1-7287cd98d0d4/e93bd390-0caa-6c00-f03a-52237a196617/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/4b718bb9cc7d41fbac6d43fb66a6909e 2024-12-06T01:45:46,463 INFO [M:0;00c764f7c3aa:37657 {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/eff72b7a-3322-3fdb-88e1-7287cd98d0d4/e93bd390-0caa-6c00-f03a-52237a196617/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/4b718bb9cc7d41fbac6d43fb66a6909e, entries=11, sequenceid=102, filesize=6.5 K 2024-12-06T01:45:46,464 DEBUG [M:0;00c764f7c3aa:37657 {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/eff72b7a-3322-3fdb-88e1-7287cd98d0d4/e93bd390-0caa-6c00-f03a-52237a196617/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/ab53492d199048588c48c872c891d5b7 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/eff72b7a-3322-3fdb-88e1-7287cd98d0d4/e93bd390-0caa-6c00-f03a-52237a196617/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/ab53492d199048588c48c872c891d5b7 2024-12-06T01:45:46,465 INFO [M:0;00c764f7c3aa:37657 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for ab53492d199048588c48c872c891d5b7 2024-12-06T01:45:46,465 INFO [M:0;00c764f7c3aa:37657 {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/eff72b7a-3322-3fdb-88e1-7287cd98d0d4/e93bd390-0caa-6c00-f03a-52237a196617/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/ab53492d199048588c48c872c891d5b7, entries=3, sequenceid=102, filesize=5.3 K 2024-12-06T01:45:46,466 INFO [M:0;00c764f7c3aa:37657 {}] regionserver.HRegion(3140): Finished flush of dataSize ~39.50 KB/40448, heapSize ~49.61 KB/50800, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 81ms, sequenceid=102, compaction requested=false 2024-12-06T01:45:46,468 INFO [M:0;00c764f7c3aa:37657 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-06T01:45:46,468 DEBUG [M:0;00c764f7c3aa:37657 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1733449546385Disabling compacts and flushes for region at 1733449546385Disabling writes for close at 1733449546385Obtaining lock to block concurrent updates at 1733449546385Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1733449546385Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=40448, getHeapSize=51040, getOffHeapSize=0, getCellsCount=127 at 1733449546385Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1733449546386 (+1 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1733449546386Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1733449546409 (+23 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1733449546409Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1733449546412 (+3 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1733449546433 (+21 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1733449546434 (+1 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1733449546436 (+2 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1733449546456 (+20 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1733449546456Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@3e3119b7: reopening flushed file at 1733449546460 (+4 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@19127245: reopening flushed file at 1733449546461 (+1 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@65dc9197: reopening flushed file at 1733449546463 (+2 ms)Finished flush of dataSize ~39.50 KB/40448, heapSize ~49.61 KB/50800, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 81ms, sequenceid=102, compaction requested=false at 1733449546466 (+3 ms)Writing region close event to WAL at 1733449546468 (+2 ms)Closed at 1733449546468 2024-12-06T01:45:46,469 INFO [M:0;00c764f7c3aa:37657 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down. 2024-12-06T01:45:46,469 INFO [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-12-06T01:45:46,469 INFO [M:0;00c764f7c3aa:37657 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:37657 2024-12-06T01:45:46,469 INFO [M:0;00c764f7c3aa:37657 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-12-06T01:45:46,473 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46757-0x1006500a2f60001, quorum=127.0.0.1:61049, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-06T01:45:46,473 INFO [RS:0;00c764f7c3aa:46757 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-12-06T01:45:46,473 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46757-0x1006500a2f60001, quorum=127.0.0.1:61049, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-06T01:45:46,473 INFO [RS:0;00c764f7c3aa:46757 {}] regionserver.HRegionServer(1031): Exiting; stopping=00c764f7c3aa,46757,1733449542231; zookeeper connection closed. 2024-12-06T01:45:46,473 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@6841ed69 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@6841ed69 2024-12-06T01:45:46,473 INFO [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 3 regionserver(s) complete 2024-12-06T01:45:46,572 INFO [M:0;00c764f7c3aa:37657 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-12-06T01:45:46,572 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37657-0x1006500a2f60000, quorum=127.0.0.1:61049, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-06T01:45:46,572 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37657-0x1006500a2f60000, quorum=127.0.0.1:61049, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-06T01:45:46,574 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers 2024-12-06T01:45:46,576 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down 2024-12-06T01:45:46,576 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-12-06T01:45:46,583 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: io.hfile.TestBlockEvictionOnRegionMovement#testBlockEvictionOnGracefulStop Thread=72 (was 52) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-7-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-8-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HMaster-EventLoopGroup-6-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-7-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HMaster-EventLoopGroup-6-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HMaster-EventLoopGroup-6-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-9 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-9-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-9-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-7 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-8-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-7-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-8-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-6 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-8 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=382 (was 346) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=516 (was 527), ProcessCount=11 (was 11), AvailableMemoryMB=10021 (was 10113)