2024-11-26 07:53:39,791 main DEBUG Apache Log4j Core 2.17.2 initializing configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@5b38c1ec 2024-11-26 07:53:39,813 main DEBUG Took 0.017582 seconds to load 1 plugins from package org.apache.hadoop.hbase.logging 2024-11-26 07:53:39,814 main DEBUG PluginManager 'Core' found 129 plugins 2024-11-26 07:53:39,814 main DEBUG PluginManager 'Level' found 0 plugins 2024-11-26 07:53:39,815 main DEBUG PluginManager 'Lookup' found 16 plugins 2024-11-26 07:53:39,816 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-26 07:53:39,831 main DEBUG PluginManager 'TypeConverter' found 26 plugins 2024-11-26 07:53:39,861 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.util.MBeans", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-26 07:53:39,863 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-26 07:53:39,865 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.logging.TestJul2Slf4j", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-26 07:53:39,866 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-26 07:53:39,866 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.zookeeper", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-26 07:53:39,867 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-26 07:53:39,868 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSinkAdapter", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-26 07:53:39,868 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-26 07:53:39,869 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSystemImpl", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-26 07:53:39,869 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-26 07:53:39,871 main DEBUG LoggerConfig$Builder(additivity="false", level="WARN", levelAndRefs="null", name="org.apache.directory", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-26 07:53:39,872 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-26 07:53:39,872 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.ipc.FailedServers", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-26 07:53:39,873 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-26 07:53:39,873 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsConfig", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-26 07:53:39,874 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-26 07:53:39,875 main DEBUG LoggerConfig$Builder(additivity="null", level="INFO", levelAndRefs="null", name="org.apache.hadoop.hbase.ScheduledChore", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-26 07:53:39,875 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-26 07:53:39,876 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.regionserver.RSRpcServices", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-26 07:53:39,876 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-26 07:53:39,877 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-26 07:53:39,877 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-26 07:53:39,878 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-26 07:53:39,878 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-26 07:53:39,879 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hbase.thirdparty.io.netty.channel", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-26 07:53:39,879 main DEBUG Building Plugin[name=root, class=org.apache.logging.log4j.core.config.LoggerConfig$RootLogger]. 2024-11-26 07:53:39,881 main DEBUG LoggerConfig$RootLogger$Builder(additivity="null", level="null", levelAndRefs="INFO,Console", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-26 07:53:39,883 main DEBUG Building Plugin[name=loggers, class=org.apache.logging.log4j.core.config.LoggersPlugin]. 2024-11-26 07:53:39,885 main DEBUG createLoggers(={org.apache.hadoop.metrics2.util.MBeans, org.apache.hadoop.hbase.logging.TestJul2Slf4j, org.apache.zookeeper, org.apache.hadoop.metrics2.impl.MetricsSinkAdapter, org.apache.hadoop.metrics2.impl.MetricsSystemImpl, org.apache.directory, org.apache.hadoop.hbase.ipc.FailedServers, org.apache.hadoop.metrics2.impl.MetricsConfig, org.apache.hadoop.hbase.ScheduledChore, org.apache.hadoop.hbase.regionserver.RSRpcServices, org.apache.hadoop, org.apache.hadoop.hbase, org.apache.hbase.thirdparty.io.netty.channel, root}) 2024-11-26 07:53:39,886 main DEBUG Building Plugin[name=layout, class=org.apache.logging.log4j.core.layout.PatternLayout]. 2024-11-26 07:53:39,887 main DEBUG PatternLayout$Builder(pattern="%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n", PatternSelector=null, Configuration(PropertiesConfig), Replace=null, charset="null", alwaysWriteExceptions="null", disableAnsi="null", noConsoleNoAnsi="null", header="null", footer="null") 2024-11-26 07:53:39,890 main DEBUG PluginManager 'Converter' found 47 plugins 2024-11-26 07:53:39,901 main DEBUG Building Plugin[name=appender, class=org.apache.hadoop.hbase.logging.HBaseTestAppender]. 2024-11-26 07:53:39,904 main DEBUG HBaseTestAppender$Builder(target="SYSTEM_ERR", maxSize="1G", bufferedIo="null", bufferSize="null", immediateFlush="null", ignoreExceptions="null", PatternLayout(%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n), name="Console", Configuration(PropertiesConfig), Filter=null, ={}) 2024-11-26 07:53:39,907 main DEBUG Starting HBaseTestOutputStreamManager SYSTEM_ERR 2024-11-26 07:53:39,907 main DEBUG Building Plugin[name=appenders, class=org.apache.logging.log4j.core.config.AppendersPlugin]. 2024-11-26 07:53:39,908 main DEBUG createAppenders(={Console}) 2024-11-26 07:53:39,909 main DEBUG Configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@5b38c1ec initialized 2024-11-26 07:53:39,912 main DEBUG Starting configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@5b38c1ec 2024-11-26 07:53:39,912 main DEBUG Started configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@5b38c1ec OK. 2024-11-26 07:53:39,913 main DEBUG Shutting down OutputStreamManager SYSTEM_OUT.false.false-1 2024-11-26 07:53:39,914 main DEBUG OutputStream closed 2024-11-26 07:53:39,914 main DEBUG Shut down OutputStreamManager SYSTEM_OUT.false.false-1, all resources released: true 2024-11-26 07:53:39,915 main DEBUG Appender DefaultConsole-1 stopped with status true 2024-11-26 07:53:39,915 main DEBUG Stopped org.apache.logging.log4j.core.config.DefaultConfiguration@bae7dc0 OK 2024-11-26 07:53:40,015 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6 2024-11-26 07:53:40,018 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=StatusLogger 2024-11-26 07:53:40,020 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=ContextSelector 2024-11-26 07:53:40,021 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name= 2024-11-26 07:53:40,022 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.directory 2024-11-26 07:53:40,022 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSinkAdapter 2024-11-26 07:53:40,023 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.zookeeper 2024-11-26 07:53:40,023 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.logging.TestJul2Slf4j 2024-11-26 07:53:40,024 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSystemImpl 2024-11-26 07:53:40,024 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.util.MBeans 2024-11-26 07:53:40,025 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase 2024-11-26 07:53:40,025 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop 2024-11-26 07:53:40,026 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ipc.FailedServers 2024-11-26 07:53:40,026 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.regionserver.RSRpcServices 2024-11-26 07:53:40,027 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsConfig 2024-11-26 07:53:40,027 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hbase.thirdparty.io.netty.channel 2024-11-26 07:53:40,027 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ScheduledChore 2024-11-26 07:53:40,029 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Appenders,name=Console 2024-11-26 07:53:40,032 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-26 07:53:40,032 main DEBUG Reconfiguration complete for context[name=1dbd16a6] at URI jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-logging/target/hbase-logging-4.0.0-alpha-1-SNAPSHOT-tests.jar!/log4j2.properties (org.apache.logging.log4j.core.LoggerContext@2b76ff4e) with optional ClassLoader: null 2024-11-26 07:53:40,033 main DEBUG Shutdown hook enabled. Registering a new one. 2024-11-26 07:53:40,034 main DEBUG LoggerContext[name=1dbd16a6, org.apache.logging.log4j.core.LoggerContext@2b76ff4e] started OK. 2024-11-26T07:53:40,051 INFO [main {}] hbase.HBaseClassTestRule(94): Test class org.apache.hadoop.hbase.io.hfile.TestBlockEvictionOnRegionMovement timeout: 13 mins 2024-11-26 07:53:40,055 main DEBUG AsyncLogger.ThreadNameStrategy=UNCACHED (user specified null, default is UNCACHED) 2024-11-26 07:53:40,056 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-26T07:53:40,411 DEBUG [main {}] hbase.HBaseTestingUtil(323): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/480cd986-aef2-fe93-7892-18b5e497e91c 2024-11-26T07:53:40,449 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: io.hfile.TestBlockEvictionOnRegionMovement#testBlockEvictionOnRegionMove Thread=12, OpenFileDescriptor=287, MaxFileDescriptor=1048576, SystemLoadAverage=407, ProcessCount=11, AvailableMemoryMB=11334 2024-11-26T07:53:40,608 WARN [Time-limited test {}] util.NativeCodeLoader(60): Unable to load native-hadoop library for your platform... using builtin-java classes where applicable 2024-11-26T07:53:40,759 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-26T07:53:40,787 INFO [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/480cd986-aef2-fe93-7892-18b5e497e91c/cluster_ab4db52e-ed38-840d-1c1d-5bf4be8d05c2, deleteOnExit=true 2024-11-26T07:53:40,990 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/480cd986-aef2-fe93-7892-18b5e497e91c/cluster_ab4db52e-ed38-840d-1c1d-5bf4be8d05c2/zookeeper_0, clientPort=53139, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/480cd986-aef2-fe93-7892-18b5e497e91c/cluster_ab4db52e-ed38-840d-1c1d-5bf4be8d05c2/zookeeper_0/version-2, dataDirSize=457 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/480cd986-aef2-fe93-7892-18b5e497e91c/cluster_ab4db52e-ed38-840d-1c1d-5bf4be8d05c2/zookeeper_0/version-2, dataLogSize=457 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0 2024-11-26T07:53:41,006 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=53139 2024-11-26T07:53:41,007 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-26T07:53:41,102 INFO [Time-limited test {}] util.FSUtils(489): Created version file at /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/480cd986-aef2-fe93-7892-18b5e497e91c/293d53df-6485-edb9-9b39-cf743ea8b2f2 with version=8 2024-11-26T07:53:41,102 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1139): Setting hbase.fs.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/480cd986-aef2-fe93-7892-18b5e497e91c/293d53df-6485-edb9-9b39-cf743ea8b2f2/hbase-staging 2024-11-26T07:53:41,201 DEBUG [Time-limited test {}] channel.MultithreadEventLoopGroup(44): -Dio.netty.eventLoopThreads: 16 2024-11-26T07:53:41,499 INFO [Time-limited test {}] client.ConnectionUtils(128): master/719ac8e87449:0 server-side Connection retries=45 2024-11-26T07:53:41,509 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-26T07:53:41,511 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-26T07:53:41,523 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-26T07:53:41,523 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-26T07:53:41,523 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-26T07:53:41,678 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService 2024-11-26T07:53:41,766 WARN [Time-limited test {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-26T07:53:41,811 INFO [Time-limited test {}] metrics.MetricRegistriesLoader(60): Loaded MetricRegistries class org.apache.hadoop.hbase.metrics.impl.MetricRegistriesImpl 2024-11-26T07:53:41,831 DEBUG [Time-limited test {}] util.ClassSize(228): Using Unsafe to estimate memory layout 2024-11-26T07:53:41,837 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-26T07:53:41,891 DEBUG [Time-limited test {}] channel.DefaultChannelId(84): -Dio.netty.processId: 93061 (auto-detected) 2024-11-26T07:53:41,892 DEBUG [Time-limited test {}] channel.DefaultChannelId(106): -Dio.netty.machineId: 02:42:ac:ff:fe:11:00:02 (auto-detected) 2024-11-26T07:53:41,920 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:36589 2024-11-26T07:53:41,951 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:36589 connecting to ZooKeeper ensemble=127.0.0.1:53139 2024-11-26T07:53:42,052 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:365890x0, quorum=127.0.0.1:53139, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-26T07:53:42,055 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:36589-0x101776bca1c0000 connected 2024-11-26T07:53:42,139 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-26T07:53:42,140 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-26T07:53:42,149 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:36589-0x101776bca1c0000, quorum=127.0.0.1:53139, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-26T07:53:42,154 INFO [Time-limited test {}] master.HMaster(525): hbase.rootdir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/480cd986-aef2-fe93-7892-18b5e497e91c/293d53df-6485-edb9-9b39-cf743ea8b2f2, hbase.cluster.distributed=false 2024-11-26T07:53:42,182 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:36589-0x101776bca1c0000, quorum=127.0.0.1:53139, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-26T07:53:42,192 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=36589 2024-11-26T07:53:42,194 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=36589 2024-11-26T07:53:42,198 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=36589 2024-11-26T07:53:42,199 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=36589 2024-11-26T07:53:42,201 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=36589 2024-11-26T07:53:42,304 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/719ac8e87449:0 server-side Connection retries=45 2024-11-26T07:53:42,306 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-26T07:53:42,307 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-26T07:53:42,308 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-26T07:53:42,308 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-26T07:53:42,309 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-26T07:53:42,313 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-26T07:53:42,317 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-26T07:53:42,319 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:37023 2024-11-26T07:53:42,322 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:37023 connecting to ZooKeeper ensemble=127.0.0.1:53139 2024-11-26T07:53:42,323 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-26T07:53:42,324 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-26T07:53:42,376 DEBUG [pool-14-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:370230x0, quorum=127.0.0.1:53139, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-26T07:53:42,377 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:37023-0x101776bca1c0001 connected 2024-11-26T07:53:42,381 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:37023-0x101776bca1c0001, quorum=127.0.0.1:53139, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-26T07:53:42,387 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-26T07:53:42,410 INFO [Time-limited test {}] bucket.FileIOEngine(99): Allocating cache 400 MB, on the path:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/480cd986-aef2-fe93-7892-18b5e497e91c/bucket.cache 2024-11-26T07:53:42,418 INFO [Time-limited test {}] bucket.BucketCache(364): Instantiating BucketCache with acceptableFactor: 0.95, minFactor: 0.85, extraFreeFactor: 0.1, singleFactor: 0.25, multiFactor: 0.5, memoryFactor: 0.25, useStrongRef: false 2024-11-26T07:53:42,420 INFO [Time-limited test {}] bucket.BucketCache(478): Starting BucketCachePersisterThread 2024-11-26T07:53:42,423 INFO [Time-limited test {}] bucket.BucketCachePersister(38): BucketCachePersister started with interval: 100 2024-11-26T07:53:42,435 INFO [Thread-10 {}] bucket.BucketCache(1530): Started retrieving bucket cache from file 2024-11-26T07:53:42,435 WARN [Thread-10 {}] bucket.BucketCache(1533): Persistence file missing! It's ok if it's first run after enabling persistent cache. 2024-11-26T07:53:42,448 INFO [Time-limited test {}] bucket.BucketCache(403): Started bucket cache; ioengine=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/480cd986-aef2-fe93-7892-18b5e497e91c/bucket.cache, capacity=400 MB, blockSize=64 KB, writerThreadNum=3, writerQLen=64, persistencePath=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/480cd986-aef2-fe93-7892-18b5e497e91c/bucket.persistence, bucketAllocator=org.apache.hadoop.hbase.io.hfile.bucket.BucketAllocator 2024-11-26T07:53:42,449 INFO [Thread-10 {}] bucket.BucketAllocator(343): Cache totalSize=418148352, buckets=199, bucket capacity=2101248=(4*525312)=(FEWEST_ITEMS_IN_BUCKET*(largest configured bucketcache size)) 2024-11-26T07:53:42,449 INFO [Thread-10 {}] bucket.BucketCache(413): Persistent bucket cache recovery from /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/480cd986-aef2-fe93-7892-18b5e497e91c/bucket.persistence is complete. 2024-11-26T07:53:42,455 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-26T07:53:42,459 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:37023-0x101776bca1c0001, quorum=127.0.0.1:53139, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-26T07:53:42,465 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:37023-0x101776bca1c0001, quorum=127.0.0.1:53139, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-26T07:53:42,475 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=37023 2024-11-26T07:53:42,476 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=37023 2024-11-26T07:53:42,481 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=37023 2024-11-26T07:53:42,483 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=37023 2024-11-26T07:53:42,489 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=37023 2024-11-26T07:53:42,518 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/719ac8e87449:0 server-side Connection retries=45 2024-11-26T07:53:42,518 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-26T07:53:42,519 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-26T07:53:42,519 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-26T07:53:42,519 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-26T07:53:42,520 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-26T07:53:42,520 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-26T07:53:42,521 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-26T07:53:42,523 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:40465 2024-11-26T07:53:42,526 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:40465 connecting to ZooKeeper ensemble=127.0.0.1:53139 2024-11-26T07:53:42,527 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-26T07:53:42,528 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-26T07:53:42,552 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:404650x0, quorum=127.0.0.1:53139, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-26T07:53:42,553 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:404650x0, quorum=127.0.0.1:53139, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-26T07:53:42,554 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-26T07:53:42,555 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:40465-0x101776bca1c0002 connected 2024-11-26T07:53:42,556 INFO [Time-limited test {}] bucket.FileIOEngine(99): Allocating cache 400 MB, on the path:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/480cd986-aef2-fe93-7892-18b5e497e91c/bucket.cache 2024-11-26T07:53:42,557 INFO [Time-limited test {}] bucket.BucketCache(364): Instantiating BucketCache with acceptableFactor: 0.95, minFactor: 0.85, extraFreeFactor: 0.1, singleFactor: 0.25, multiFactor: 0.5, memoryFactor: 0.25, useStrongRef: false 2024-11-26T07:53:42,557 INFO [Time-limited test {}] bucket.BucketCache(478): Starting BucketCachePersisterThread 2024-11-26T07:53:42,557 INFO [Time-limited test {}] bucket.BucketCachePersister(38): BucketCachePersister started with interval: 100 2024-11-26T07:53:42,562 INFO [Thread-11 {}] bucket.BucketCache(1530): Started retrieving bucket cache from file 2024-11-26T07:53:42,563 WARN [Thread-11 {}] bucket.BucketCache(1533): Persistence file missing! It's ok if it's first run after enabling persistent cache. 2024-11-26T07:53:42,565 INFO [Thread-11 {}] bucket.BucketAllocator(343): Cache totalSize=418148352, buckets=199, bucket capacity=2101248=(4*525312)=(FEWEST_ITEMS_IN_BUCKET*(largest configured bucketcache size)) 2024-11-26T07:53:42,565 INFO [Thread-11 {}] bucket.BucketCache(413): Persistent bucket cache recovery from /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/480cd986-aef2-fe93-7892-18b5e497e91c/bucket.persistence is complete. 2024-11-26T07:53:42,566 INFO [Time-limited test {}] bucket.BucketCache(403): Started bucket cache; ioengine=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/480cd986-aef2-fe93-7892-18b5e497e91c/bucket.cache, capacity=400 MB, blockSize=64 KB, writerThreadNum=3, writerQLen=64, persistencePath=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/480cd986-aef2-fe93-7892-18b5e497e91c/bucket.persistence, bucketAllocator=org.apache.hadoop.hbase.io.hfile.bucket.BucketAllocator 2024-11-26T07:53:42,569 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-26T07:53:42,571 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:40465-0x101776bca1c0002, quorum=127.0.0.1:53139, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-26T07:53:42,575 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:40465-0x101776bca1c0002, quorum=127.0.0.1:53139, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-26T07:53:42,588 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=40465 2024-11-26T07:53:42,588 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=40465 2024-11-26T07:53:42,591 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=40465 2024-11-26T07:53:42,592 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=40465 2024-11-26T07:53:42,597 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=40465 2024-11-26T07:53:42,625 DEBUG [M:0;719ac8e87449:36589 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;719ac8e87449:36589 2024-11-26T07:53:42,627 INFO [master/719ac8e87449:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/719ac8e87449,36589,1732607621309 2024-11-26T07:53:42,660 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36589-0x101776bca1c0000, quorum=127.0.0.1:53139, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-26T07:53:42,660 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40465-0x101776bca1c0002, quorum=127.0.0.1:53139, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-26T07:53:42,661 DEBUG [pool-14-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37023-0x101776bca1c0001, quorum=127.0.0.1:53139, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-26T07:53:42,665 DEBUG [master/719ac8e87449:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:36589-0x101776bca1c0000, quorum=127.0.0.1:53139, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/719ac8e87449,36589,1732607621309 2024-11-26T07:53:42,709 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40465-0x101776bca1c0002, quorum=127.0.0.1:53139, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-26T07:53:42,710 DEBUG [pool-14-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37023-0x101776bca1c0001, quorum=127.0.0.1:53139, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-26T07:53:42,710 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40465-0x101776bca1c0002, quorum=127.0.0.1:53139, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-26T07:53:42,710 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36589-0x101776bca1c0000, quorum=127.0.0.1:53139, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-26T07:53:42,712 DEBUG [pool-14-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37023-0x101776bca1c0001, quorum=127.0.0.1:53139, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-26T07:53:42,715 DEBUG [master/719ac8e87449:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:36589-0x101776bca1c0000, quorum=127.0.0.1:53139, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-11-26T07:53:42,717 INFO [master/719ac8e87449:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/719ac8e87449,36589,1732607621309 from backup master directory 2024-11-26T07:53:42,734 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40465-0x101776bca1c0002, quorum=127.0.0.1:53139, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-26T07:53:42,735 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36589-0x101776bca1c0000, quorum=127.0.0.1:53139, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/719ac8e87449,36589,1732607621309 2024-11-26T07:53:42,735 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36589-0x101776bca1c0000, quorum=127.0.0.1:53139, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-26T07:53:42,737 DEBUG [pool-14-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37023-0x101776bca1c0001, quorum=127.0.0.1:53139, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-26T07:53:42,745 WARN [master/719ac8e87449:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-26T07:53:42,746 INFO [master/719ac8e87449:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=719ac8e87449,36589,1732607621309 2024-11-26T07:53:42,749 INFO [master/719ac8e87449:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating data MemStoreChunkPool with chunk size 2 MB, max count 396, initial count 0 2024-11-26T07:53:42,765 INFO [master/719ac8e87449:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating index MemStoreChunkPool with chunk size 204.80 KB, max count 440, initial count 0 2024-11-26T07:53:42,792 DEBUG [master/719ac8e87449:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/480cd986-aef2-fe93-7892-18b5e497e91c/293d53df-6485-edb9-9b39-cf743ea8b2f2/hbase.id] with ID: 4d71bb38-f6cf-415f-a628-77b4c190248b 2024-11-26T07:53:42,792 DEBUG [master/719ac8e87449:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/480cd986-aef2-fe93-7892-18b5e497e91c/293d53df-6485-edb9-9b39-cf743ea8b2f2/.tmp/hbase.id 2024-11-26T07:53:42,828 DEBUG [master/719ac8e87449:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/480cd986-aef2-fe93-7892-18b5e497e91c/293d53df-6485-edb9-9b39-cf743ea8b2f2/.tmp/hbase.id]:[file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/480cd986-aef2-fe93-7892-18b5e497e91c/293d53df-6485-edb9-9b39-cf743ea8b2f2/hbase.id] 2024-11-26T07:53:42,904 DEBUG [master/719ac8e87449:0:becomeActiveMaster {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-26T07:53:42,911 INFO [master/719ac8e87449:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem. 2024-11-26T07:53:42,939 INFO [master/719ac8e87449:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 25ms. 2024-11-26T07:53:42,959 DEBUG [pool-14-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37023-0x101776bca1c0001, quorum=127.0.0.1:53139, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-26T07:53:42,959 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40465-0x101776bca1c0002, quorum=127.0.0.1:53139, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-26T07:53:42,960 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36589-0x101776bca1c0000, quorum=127.0.0.1:53139, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-26T07:53:43,000 INFO [master/719ac8e87449:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-26T07:53:43,004 INFO [master/719ac8e87449:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-11-26T07:53:43,055 DEBUG [master/719ac8e87449:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(244): No decryptEncryptedDataEncryptionKey method in DFSClient, should be hadoop version with HDFS-12396 java.lang.NoSuchMethodException: org.apache.hadoop.hdfs.DFSClient.decryptEncryptedDataEncryptionKey(org.apache.hadoop.fs.FileEncryptionInfo) at java.lang.Class.getDeclaredMethod(Class.java:2675) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelperWithoutHDFS12396(FanOutOneBlockAsyncDFSOutputSaslHelper.java:183) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelper(FanOutOneBlockAsyncDFSOutputSaslHelper.java:242) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.(FanOutOneBlockAsyncDFSOutputSaslHelper.java:253) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.lang.Class.forName0(Native Method) ~[?:?] at java.lang.Class.forName(Class.java:375) ~[?:?] at org.apache.hadoop.hbase.wal.AsyncFSWALProvider.load(AsyncFSWALProvider.java:150) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.getProviderClass(WALFactory.java:174) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:262) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:231) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegion.create(MasterRegion.java:400) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegionFactory.create(MasterRegionFactory.java:135) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.finishActiveMasterInitialization(HMaster.java:1003) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.startActiveMasterManager(HMaster.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.lambda$run$0(HMaster.java:613) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.lambda$tracedRunnable$2(TraceUtil.java:155) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T07:53:43,061 INFO [master/719ac8e87449:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-26T07:53:43,126 INFO [master/719ac8e87449:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/480cd986-aef2-fe93-7892-18b5e497e91c/293d53df-6485-edb9-9b39-cf743ea8b2f2/MasterData/data/master/store 2024-11-26T07:53:43,161 INFO [master/719ac8e87449:0:becomeActiveMaster {}] throttle.StoreHotnessProtector(112): StoreHotnessProtector is disabled. Set hbase.region.store.parallel.put.limit > 0 to enable, which may help mitigate load under heavy write pressure. 2024-11-26T07:53:43,166 DEBUG [master/719ac8e87449:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-26T07:53:43,171 DEBUG [master/719ac8e87449:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-26T07:53:43,172 INFO [master/719ac8e87449:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-26T07:53:43,172 DEBUG [master/719ac8e87449:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-26T07:53:43,174 DEBUG [master/719ac8e87449:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-26T07:53:43,175 DEBUG [master/719ac8e87449:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-26T07:53:43,175 INFO [master/719ac8e87449:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-26T07:53:43,176 DEBUG [master/719ac8e87449:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1732607623168Disabling compacts and flushes for region at 1732607623171 (+3 ms)Disabling writes for close at 1732607623175 (+4 ms)Writing region close event to WAL at 1732607623175Closed at 1732607623175 2024-11-26T07:53:43,177 WARN [master/719ac8e87449:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/480cd986-aef2-fe93-7892-18b5e497e91c/293d53df-6485-edb9-9b39-cf743ea8b2f2/MasterData/data/master/store/.initializing 2024-11-26T07:53:43,177 DEBUG [master/719ac8e87449:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/480cd986-aef2-fe93-7892-18b5e497e91c/293d53df-6485-edb9-9b39-cf743ea8b2f2/MasterData/WALs/719ac8e87449,36589,1732607621309 2024-11-26T07:53:43,202 INFO [master/719ac8e87449:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-26T07:53:43,223 INFO [master/719ac8e87449:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=719ac8e87449%2C36589%2C1732607621309, suffix=, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/480cd986-aef2-fe93-7892-18b5e497e91c/293d53df-6485-edb9-9b39-cf743ea8b2f2/MasterData/WALs/719ac8e87449,36589,1732607621309, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/480cd986-aef2-fe93-7892-18b5e497e91c/293d53df-6485-edb9-9b39-cf743ea8b2f2/MasterData/oldWALs, maxLogs=10 2024-11-26T07:53:43,273 INFO [master/719ac8e87449:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/480cd986-aef2-fe93-7892-18b5e497e91c/293d53df-6485-edb9-9b39-cf743ea8b2f2/MasterData/WALs/719ac8e87449,36589,1732607621309/719ac8e87449%2C36589%2C1732607621309.1732607623231 2024-11-26T07:53:43,274 DEBUG [master/719ac8e87449:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-11-26T07:53:43,275 DEBUG [master/719ac8e87449:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-11-26T07:53:43,276 DEBUG [master/719ac8e87449:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-26T07:53:43,280 DEBUG [master/719ac8e87449:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-11-26T07:53:43,282 DEBUG [master/719ac8e87449:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-11-26T07:53:43,336 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-11-26T07:53:43,394 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-11-26T07:53:43,401 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T07:53:43,403 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-26T07:53:43,404 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-11-26T07:53:43,414 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-11-26T07:53:43,414 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T07:53:43,415 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-26T07:53:43,415 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-11-26T07:53:43,427 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-11-26T07:53:43,427 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T07:53:43,428 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-26T07:53:43,428 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-11-26T07:53:43,441 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-11-26T07:53:43,441 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T07:53:43,442 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-26T07:53:43,442 DEBUG [master/719ac8e87449:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682 2024-11-26T07:53:43,448 DEBUG [master/719ac8e87449:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/480cd986-aef2-fe93-7892-18b5e497e91c/293d53df-6485-edb9-9b39-cf743ea8b2f2/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-26T07:53:43,448 DEBUG [master/719ac8e87449:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/480cd986-aef2-fe93-7892-18b5e497e91c/293d53df-6485-edb9-9b39-cf743ea8b2f2/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-26T07:53:43,452 DEBUG [master/719ac8e87449:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682 2024-11-26T07:53:43,453 DEBUG [master/719ac8e87449:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682 2024-11-26T07:53:43,459 DEBUG [master/719ac8e87449:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-26T07:53:43,461 DEBUG [master/719ac8e87449:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-11-26T07:53:43,471 DEBUG [master/719ac8e87449:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/480cd986-aef2-fe93-7892-18b5e497e91c/293d53df-6485-edb9-9b39-cf743ea8b2f2/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-26T07:53:43,472 INFO [master/719ac8e87449:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=66937533, jitterRate=-0.002553030848503113}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-26T07:53:43,483 DEBUG [master/719ac8e87449:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682: Writing region info on filesystem at 1732607623299Initializing all the Stores at 1732607623300 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732607623301 (+1 ms)Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732607623308 (+7 ms)Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732607623309 (+1 ms)Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732607623309Cleaning up temporary data from old regions at 1732607623453 (+144 ms)Region opened successfully at 1732607623482 (+29 ms) 2024-11-26T07:53:43,484 INFO [master/719ac8e87449:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-11-26T07:53:43,532 DEBUG [master/719ac8e87449:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@75d6bb36, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=719ac8e87449/172.17.0.2:0 2024-11-26T07:53:43,583 INFO [master/719ac8e87449:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating... 2024-11-26T07:53:43,601 INFO [master/719ac8e87449:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5 2024-11-26T07:53:43,601 INFO [master/719ac8e87449:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50 2024-11-26T07:53:43,605 INFO [master/719ac8e87449:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-26T07:53:43,608 INFO [master/719ac8e87449:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-26T07:53:43,629 INFO [master/719ac8e87449:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 19 msec 2024-11-26T07:53:43,630 INFO [master/719ac8e87449:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-11-26T07:53:43,713 INFO [master/719ac8e87449:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2024-11-26T07:53:43,733 DEBUG [master/719ac8e87449:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:36589-0x101776bca1c0000, quorum=127.0.0.1:53139, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error) 2024-11-26T07:53:43,784 DEBUG [master/719ac8e87449:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false 2024-11-26T07:53:43,786 INFO [master/719ac8e87449:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2024-11-26T07:53:43,788 DEBUG [master/719ac8e87449:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:36589-0x101776bca1c0000, quorum=127.0.0.1:53139, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error) 2024-11-26T07:53:43,792 DEBUG [master/719ac8e87449:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false 2024-11-26T07:53:43,795 INFO [master/719ac8e87449:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2024-11-26T07:53:43,799 DEBUG [master/719ac8e87449:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:36589-0x101776bca1c0000, quorum=127.0.0.1:53139, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error) 2024-11-26T07:53:43,809 DEBUG [master/719ac8e87449:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false 2024-11-26T07:53:43,811 DEBUG [master/719ac8e87449:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:36589-0x101776bca1c0000, quorum=127.0.0.1:53139, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error) 2024-11-26T07:53:43,817 DEBUG [master/719ac8e87449:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false 2024-11-26T07:53:43,839 DEBUG [master/719ac8e87449:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:36589-0x101776bca1c0000, quorum=127.0.0.1:53139, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error) 2024-11-26T07:53:43,850 DEBUG [master/719ac8e87449:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false 2024-11-26T07:53:43,867 DEBUG [pool-14-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37023-0x101776bca1c0001, quorum=127.0.0.1:53139, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-26T07:53:43,867 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40465-0x101776bca1c0002, quorum=127.0.0.1:53139, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-26T07:53:43,868 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36589-0x101776bca1c0000, quorum=127.0.0.1:53139, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-26T07:53:43,868 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40465-0x101776bca1c0002, quorum=127.0.0.1:53139, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-26T07:53:43,868 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36589-0x101776bca1c0000, quorum=127.0.0.1:53139, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-26T07:53:43,868 DEBUG [pool-14-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37023-0x101776bca1c0001, quorum=127.0.0.1:53139, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-26T07:53:43,872 INFO [master/719ac8e87449:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=719ac8e87449,36589,1732607621309, sessionid=0x101776bca1c0000, setting cluster-up flag (Was=false) 2024-11-26T07:53:43,901 DEBUG [pool-14-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37023-0x101776bca1c0001, quorum=127.0.0.1:53139, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-26T07:53:43,901 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40465-0x101776bca1c0002, quorum=127.0.0.1:53139, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-26T07:53:43,901 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36589-0x101776bca1c0000, quorum=127.0.0.1:53139, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-26T07:53:43,925 DEBUG [master/719ac8e87449:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort 2024-11-26T07:53:43,928 DEBUG [master/719ac8e87449:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=719ac8e87449,36589,1732607621309 2024-11-26T07:53:43,942 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36589-0x101776bca1c0000, quorum=127.0.0.1:53139, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-26T07:53:43,942 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40465-0x101776bca1c0002, quorum=127.0.0.1:53139, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-26T07:53:43,942 DEBUG [pool-14-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37023-0x101776bca1c0001, quorum=127.0.0.1:53139, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-26T07:53:43,967 DEBUG [master/719ac8e87449:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort 2024-11-26T07:53:43,969 DEBUG [master/719ac8e87449:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=719ac8e87449,36589,1732607621309 2024-11-26T07:53:43,975 INFO [master/719ac8e87449:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/480cd986-aef2-fe93-7892-18b5e497e91c/293d53df-6485-edb9-9b39-cf743ea8b2f2/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again 2024-11-26T07:53:44,005 INFO [RS:1;719ac8e87449:40465 {}] regionserver.HRegionServer(746): ClusterId : 4d71bb38-f6cf-415f-a628-77b4c190248b 2024-11-26T07:53:44,006 INFO [RS:0;719ac8e87449:37023 {}] regionserver.HRegionServer(746): ClusterId : 4d71bb38-f6cf-415f-a628-77b4c190248b 2024-11-26T07:53:44,008 DEBUG [RS:1;719ac8e87449:40465 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-26T07:53:44,008 DEBUG [RS:0;719ac8e87449:37023 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-26T07:53:44,022 DEBUG [RS:1;719ac8e87449:40465 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-26T07:53:44,022 DEBUG [RS:1;719ac8e87449:40465 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-26T07:53:44,034 DEBUG [RS:0;719ac8e87449:37023 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-26T07:53:44,034 DEBUG [RS:0;719ac8e87449:37023 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-26T07:53:44,043 DEBUG [RS:1;719ac8e87449:40465 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-26T07:53:44,044 DEBUG [RS:0;719ac8e87449:37023 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-26T07:53:44,044 DEBUG [RS:1;719ac8e87449:40465 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5308a059, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=719ac8e87449/172.17.0.2:0 2024-11-26T07:53:44,044 DEBUG [RS:0;719ac8e87449:37023 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@278a1bd9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=719ac8e87449/172.17.0.2:0 2024-11-26T07:53:44,061 DEBUG [master/719ac8e87449:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-26T07:53:44,068 DEBUG [RS:1;719ac8e87449:40465 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:1;719ac8e87449:40465 2024-11-26T07:53:44,073 DEBUG [RS:0;719ac8e87449:37023 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;719ac8e87449:37023 2024-11-26T07:53:44,073 INFO [RS:0;719ac8e87449:37023 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-26T07:53:44,073 INFO [RS:1;719ac8e87449:40465 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-26T07:53:44,074 INFO [RS:1;719ac8e87449:40465 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-26T07:53:44,074 INFO [RS:0;719ac8e87449:37023 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-26T07:53:44,074 DEBUG [RS:1;719ac8e87449:40465 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-26T07:53:44,074 DEBUG [RS:0;719ac8e87449:37023 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-26T07:53:44,075 INFO [master/719ac8e87449:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2 2024-11-26T07:53:44,077 INFO [RS:0;719ac8e87449:37023 {}] regionserver.HRegionServer(2659): reportForDuty to master=719ac8e87449,36589,1732607621309 with port=37023, startcode=1732607622257 2024-11-26T07:53:44,079 INFO [RS:1;719ac8e87449:40465 {}] regionserver.HRegionServer(2659): reportForDuty to master=719ac8e87449,36589,1732607621309 with port=40465, startcode=1732607622517 2024-11-26T07:53:44,087 INFO [master/719ac8e87449:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2024-11-26T07:53:44,091 DEBUG [RS:0;719ac8e87449:37023 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-26T07:53:44,091 DEBUG [RS:1;719ac8e87449:40465 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-26T07:53:44,094 DEBUG [master/719ac8e87449:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: 719ac8e87449,36589,1732607621309 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2024-11-26T07:53:44,110 DEBUG [master/719ac8e87449:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/719ac8e87449:0, corePoolSize=5, maxPoolSize=5 2024-11-26T07:53:44,110 DEBUG [master/719ac8e87449:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/719ac8e87449:0, corePoolSize=5, maxPoolSize=5 2024-11-26T07:53:44,111 DEBUG [master/719ac8e87449:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/719ac8e87449:0, corePoolSize=5, maxPoolSize=5 2024-11-26T07:53:44,111 DEBUG [master/719ac8e87449:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/719ac8e87449:0, corePoolSize=5, maxPoolSize=5 2024-11-26T07:53:44,111 DEBUG [master/719ac8e87449:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/719ac8e87449:0, corePoolSize=10, maxPoolSize=10 2024-11-26T07:53:44,111 DEBUG [master/719ac8e87449:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/719ac8e87449:0, corePoolSize=1, maxPoolSize=1 2024-11-26T07:53:44,112 DEBUG [master/719ac8e87449:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/719ac8e87449:0, corePoolSize=2, maxPoolSize=2 2024-11-26T07:53:44,112 DEBUG [master/719ac8e87449:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/719ac8e87449:0, corePoolSize=1, maxPoolSize=1 2024-11-26T07:53:44,135 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-26T07:53:44,135 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region 2024-11-26T07:53:44,141 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T07:53:44,141 INFO [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-11-26T07:53:44,160 INFO [master/719ac8e87449:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732607654160 2024-11-26T07:53:44,162 INFO [master/719ac8e87449:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2024-11-26T07:53:44,163 INFO [master/719ac8e87449:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2024-11-26T07:53:44,168 INFO [master/719ac8e87449:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2024-11-26T07:53:44,169 INFO [master/719ac8e87449:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2024-11-26T07:53:44,170 INFO [master/719ac8e87449:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2024-11-26T07:53:44,170 INFO [master/719ac8e87449:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2024-11-26T07:53:44,172 INFO [master/719ac8e87449:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-26T07:53:44,178 INFO [master/719ac8e87449:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2024-11-26T07:53:44,179 INFO [master/719ac8e87449:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2024-11-26T07:53:44,180 INFO [master/719ac8e87449:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2024-11-26T07:53:44,183 INFO [master/719ac8e87449:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2024-11-26T07:53:44,184 INFO [master/719ac8e87449:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2024-11-26T07:53:44,189 DEBUG [master/719ac8e87449:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/719ac8e87449:0:becomeActiveMaster-HFileCleaner.large.0-1732607624186,5,FailOnTimeoutGroup] 2024-11-26T07:53:44,202 DEBUG [master/719ac8e87449:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/719ac8e87449:0:becomeActiveMaster-HFileCleaner.small.0-1732607624189,5,FailOnTimeoutGroup] 2024-11-26T07:53:44,202 INFO [master/719ac8e87449:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-26T07:53:44,202 INFO [master/719ac8e87449:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2024-11-26T07:53:44,209 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37245, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=RegionServerStatusService 2024-11-26T07:53:44,210 INFO [master/719ac8e87449:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2024-11-26T07:53:44,211 INFO [master/719ac8e87449:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2024-11-26T07:53:44,218 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60245, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=RegionServerStatusService 2024-11-26T07:53:44,219 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=36589 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 719ac8e87449,40465,1732607622517 2024-11-26T07:53:44,222 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=36589 {}] master.ServerManager(517): Registering regionserver=719ac8e87449,40465,1732607622517 2024-11-26T07:53:44,231 INFO [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/480cd986-aef2-fe93-7892-18b5e497e91c/293d53df-6485-edb9-9b39-cf743ea8b2f2/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321 2024-11-26T07:53:44,232 INFO [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/480cd986-aef2-fe93-7892-18b5e497e91c/293d53df-6485-edb9-9b39-cf743ea8b2f2 2024-11-26T07:53:44,237 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=36589 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 719ac8e87449,37023,1732607622257 2024-11-26T07:53:44,240 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=36589 {}] master.ServerManager(517): Registering regionserver=719ac8e87449,37023,1732607622257 2024-11-26T07:53:44,242 DEBUG [RS:1;719ac8e87449:40465 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/480cd986-aef2-fe93-7892-18b5e497e91c/293d53df-6485-edb9-9b39-cf743ea8b2f2 2024-11-26T07:53:44,242 DEBUG [RS:1;719ac8e87449:40465 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=file:/ 2024-11-26T07:53:44,243 DEBUG [RS:1;719ac8e87449:40465 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-26T07:53:44,245 DEBUG [RS:0;719ac8e87449:37023 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/480cd986-aef2-fe93-7892-18b5e497e91c/293d53df-6485-edb9-9b39-cf743ea8b2f2 2024-11-26T07:53:44,245 DEBUG [RS:0;719ac8e87449:37023 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=file:/ 2024-11-26T07:53:44,246 DEBUG [RS:0;719ac8e87449:37023 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-26T07:53:44,268 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36589-0x101776bca1c0000, quorum=127.0.0.1:53139, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-26T07:53:44,284 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-26T07:53:44,287 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family info of region 1588230740 2024-11-26T07:53:44,296 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-26T07:53:44,296 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T07:53:44,296 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-26T07:53:44,298 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family ns of region 1588230740 2024-11-26T07:53:44,298 DEBUG [RS:0;719ac8e87449:37023 {}] zookeeper.ZKUtil(111): regionserver:37023-0x101776bca1c0001, quorum=127.0.0.1:53139, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/719ac8e87449,37023,1732607622257 2024-11-26T07:53:44,299 WARN [RS:0;719ac8e87449:37023 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-26T07:53:44,299 INFO [RS:0;719ac8e87449:37023 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-26T07:53:44,299 DEBUG [RS:1;719ac8e87449:40465 {}] zookeeper.ZKUtil(111): regionserver:40465-0x101776bca1c0002, quorum=127.0.0.1:53139, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/719ac8e87449,40465,1732607622517 2024-11-26T07:53:44,299 DEBUG [RS:0;719ac8e87449:37023 {}] regionserver.HRegionServer(1793): logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/480cd986-aef2-fe93-7892-18b5e497e91c/293d53df-6485-edb9-9b39-cf743ea8b2f2/WALs/719ac8e87449,37023,1732607622257 2024-11-26T07:53:44,299 WARN [RS:1;719ac8e87449:40465 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-26T07:53:44,300 INFO [RS:1;719ac8e87449:40465 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-26T07:53:44,300 DEBUG [RS:1;719ac8e87449:40465 {}] regionserver.HRegionServer(1793): logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/480cd986-aef2-fe93-7892-18b5e497e91c/293d53df-6485-edb9-9b39-cf743ea8b2f2/WALs/719ac8e87449,40465,1732607622517 2024-11-26T07:53:44,301 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [719ac8e87449,40465,1732607622517] 2024-11-26T07:53:44,301 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [719ac8e87449,37023,1732607622257] 2024-11-26T07:53:44,313 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-26T07:53:44,313 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T07:53:44,313 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-26T07:53:44,314 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family rep_barrier of region 1588230740 2024-11-26T07:53:44,319 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-26T07:53:44,320 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T07:53:44,320 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-26T07:53:44,321 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family table of region 1588230740 2024-11-26T07:53:44,330 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-26T07:53:44,330 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T07:53:44,330 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-26T07:53:44,331 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-26T07:53:44,332 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/480cd986-aef2-fe93-7892-18b5e497e91c/293d53df-6485-edb9-9b39-cf743ea8b2f2/data/hbase/meta/1588230740 2024-11-26T07:53:44,332 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/480cd986-aef2-fe93-7892-18b5e497e91c/293d53df-6485-edb9-9b39-cf743ea8b2f2/data/hbase/meta/1588230740 2024-11-26T07:53:44,333 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-26T07:53:44,333 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-26T07:53:44,333 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-26T07:53:44,334 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-26T07:53:44,340 INFO [RS:1;719ac8e87449:40465 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-26T07:53:44,340 INFO [RS:0;719ac8e87449:37023 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-26T07:53:44,354 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/480cd986-aef2-fe93-7892-18b5e497e91c/293d53df-6485-edb9-9b39-cf743ea8b2f2/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-26T07:53:44,355 INFO [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=68062753, jitterRate=0.014214053750038147}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-26T07:53:44,360 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740: Writing region info on filesystem at 1732607624284Initializing all the Stores at 1732607624285 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732607624285Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732607624286 (+1 ms)Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732607624286Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732607624287 (+1 ms)Cleaning up temporary data from old regions at 1732607624333 (+46 ms)Region opened successfully at 1732607624359 (+26 ms) 2024-11-26T07:53:44,360 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-26T07:53:44,360 INFO [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-26T07:53:44,360 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-26T07:53:44,360 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-26T07:53:44,361 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-26T07:53:44,364 INFO [RS:1;719ac8e87449:40465 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-26T07:53:44,367 INFO [RS:0;719ac8e87449:37023 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-26T07:53:44,371 INFO [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-26T07:53:44,372 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1732607624360Disabling compacts and flushes for region at 1732607624360Disabling writes for close at 1732607624360Writing region close event to WAL at 1732607624371 (+11 ms)Closed at 1732607624371 2024-11-26T07:53:44,373 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-26T07:53:44,374 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta 2024-11-26T07:53:44,378 INFO [RS:0;719ac8e87449:37023 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-26T07:53:44,378 INFO [RS:1;719ac8e87449:40465 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-26T07:53:44,378 INFO [RS:1;719ac8e87449:40465 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-26T07:53:44,378 INFO [RS:0;719ac8e87449:37023 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-26T07:53:44,380 INFO [RS:1;719ac8e87449:40465 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-26T07:53:44,385 INFO [RS:1;719ac8e87449:40465 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-26T07:53:44,386 INFO [RS:0;719ac8e87449:37023 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-26T07:53:44,387 INFO [RS:0;719ac8e87449:37023 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-26T07:53:44,387 INFO [RS:0;719ac8e87449:37023 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-26T07:53:44,387 INFO [RS:1;719ac8e87449:40465 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-26T07:53:44,388 DEBUG [RS:1;719ac8e87449:40465 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/719ac8e87449:0, corePoolSize=1, maxPoolSize=1 2024-11-26T07:53:44,388 DEBUG [RS:1;719ac8e87449:40465 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/719ac8e87449:0, corePoolSize=1, maxPoolSize=1 2024-11-26T07:53:44,388 DEBUG [RS:0;719ac8e87449:37023 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/719ac8e87449:0, corePoolSize=1, maxPoolSize=1 2024-11-26T07:53:44,388 DEBUG [RS:1;719ac8e87449:40465 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/719ac8e87449:0, corePoolSize=1, maxPoolSize=1 2024-11-26T07:53:44,388 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-11-26T07:53:44,388 DEBUG [RS:0;719ac8e87449:37023 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/719ac8e87449:0, corePoolSize=1, maxPoolSize=1 2024-11-26T07:53:44,388 DEBUG [RS:1;719ac8e87449:40465 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/719ac8e87449:0, corePoolSize=1, maxPoolSize=1 2024-11-26T07:53:44,389 DEBUG [RS:0;719ac8e87449:37023 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/719ac8e87449:0, corePoolSize=1, maxPoolSize=1 2024-11-26T07:53:44,389 DEBUG [RS:1;719ac8e87449:40465 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/719ac8e87449:0, corePoolSize=1, maxPoolSize=1 2024-11-26T07:53:44,389 DEBUG [RS:0;719ac8e87449:37023 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/719ac8e87449:0, corePoolSize=1, maxPoolSize=1 2024-11-26T07:53:44,389 DEBUG [RS:1;719ac8e87449:40465 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/719ac8e87449:0, corePoolSize=2, maxPoolSize=2 2024-11-26T07:53:44,389 DEBUG [RS:0;719ac8e87449:37023 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/719ac8e87449:0, corePoolSize=1, maxPoolSize=1 2024-11-26T07:53:44,389 DEBUG [RS:1;719ac8e87449:40465 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/719ac8e87449:0, corePoolSize=1, maxPoolSize=1 2024-11-26T07:53:44,389 DEBUG [RS:0;719ac8e87449:37023 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/719ac8e87449:0, corePoolSize=2, maxPoolSize=2 2024-11-26T07:53:44,389 DEBUG [RS:1;719ac8e87449:40465 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/719ac8e87449:0, corePoolSize=1, maxPoolSize=1 2024-11-26T07:53:44,389 DEBUG [RS:0;719ac8e87449:37023 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/719ac8e87449:0, corePoolSize=1, maxPoolSize=1 2024-11-26T07:53:44,389 DEBUG [RS:1;719ac8e87449:40465 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/719ac8e87449:0, corePoolSize=1, maxPoolSize=1 2024-11-26T07:53:44,390 DEBUG [RS:0;719ac8e87449:37023 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/719ac8e87449:0, corePoolSize=1, maxPoolSize=1 2024-11-26T07:53:44,390 DEBUG [RS:1;719ac8e87449:40465 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/719ac8e87449:0, corePoolSize=1, maxPoolSize=1 2024-11-26T07:53:44,390 DEBUG [RS:0;719ac8e87449:37023 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/719ac8e87449:0, corePoolSize=1, maxPoolSize=1 2024-11-26T07:53:44,390 DEBUG [RS:1;719ac8e87449:40465 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/719ac8e87449:0, corePoolSize=1, maxPoolSize=1 2024-11-26T07:53:44,390 DEBUG [RS:0;719ac8e87449:37023 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/719ac8e87449:0, corePoolSize=1, maxPoolSize=1 2024-11-26T07:53:44,390 DEBUG [RS:1;719ac8e87449:40465 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/719ac8e87449:0, corePoolSize=1, maxPoolSize=1 2024-11-26T07:53:44,390 DEBUG [RS:0;719ac8e87449:37023 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/719ac8e87449:0, corePoolSize=1, maxPoolSize=1 2024-11-26T07:53:44,390 DEBUG [RS:1;719ac8e87449:40465 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/719ac8e87449:0, corePoolSize=3, maxPoolSize=3 2024-11-26T07:53:44,390 DEBUG [RS:0;719ac8e87449:37023 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/719ac8e87449:0, corePoolSize=1, maxPoolSize=1 2024-11-26T07:53:44,390 DEBUG [RS:1;719ac8e87449:40465 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/719ac8e87449:0, corePoolSize=3, maxPoolSize=3 2024-11-26T07:53:44,391 DEBUG [RS:0;719ac8e87449:37023 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/719ac8e87449:0, corePoolSize=3, maxPoolSize=3 2024-11-26T07:53:44,391 DEBUG [RS:0;719ac8e87449:37023 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/719ac8e87449:0, corePoolSize=3, maxPoolSize=3 2024-11-26T07:53:44,395 INFO [RS:1;719ac8e87449:40465 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-26T07:53:44,395 INFO [RS:1;719ac8e87449:40465 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-26T07:53:44,395 INFO [RS:1;719ac8e87449:40465 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-26T07:53:44,395 INFO [RS:1;719ac8e87449:40465 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-26T07:53:44,396 INFO [RS:1;719ac8e87449:40465 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-26T07:53:44,396 INFO [RS:1;719ac8e87449:40465 {}] hbase.ChoreService(168): Chore ScheduledChore name=719ac8e87449,40465,1732607622517-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-26T07:53:44,399 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-26T07:53:44,403 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2024-11-26T07:53:44,411 INFO [RS:0;719ac8e87449:37023 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-26T07:53:44,411 INFO [RS:0;719ac8e87449:37023 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-26T07:53:44,411 INFO [RS:0;719ac8e87449:37023 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-26T07:53:44,411 INFO [RS:0;719ac8e87449:37023 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-26T07:53:44,411 INFO [RS:0;719ac8e87449:37023 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-26T07:53:44,411 INFO [RS:0;719ac8e87449:37023 {}] hbase.ChoreService(168): Chore ScheduledChore name=719ac8e87449,37023,1732607622257-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-26T07:53:44,428 INFO [RS:1;719ac8e87449:40465 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-26T07:53:44,430 INFO [RS:1;719ac8e87449:40465 {}] hbase.ChoreService(168): Chore ScheduledChore name=719ac8e87449,40465,1732607622517-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-26T07:53:44,431 INFO [RS:1;719ac8e87449:40465 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-26T07:53:44,431 INFO [RS:1;719ac8e87449:40465 {}] regionserver.Replication(171): 719ac8e87449,40465,1732607622517 started 2024-11-26T07:53:44,436 INFO [RS:0;719ac8e87449:37023 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-26T07:53:44,436 INFO [RS:0;719ac8e87449:37023 {}] hbase.ChoreService(168): Chore ScheduledChore name=719ac8e87449,37023,1732607622257-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-26T07:53:44,437 INFO [RS:0;719ac8e87449:37023 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-26T07:53:44,437 INFO [RS:0;719ac8e87449:37023 {}] regionserver.Replication(171): 719ac8e87449,37023,1732607622257 started 2024-11-26T07:53:44,456 INFO [RS:1;719ac8e87449:40465 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-26T07:53:44,456 INFO [RS:1;719ac8e87449:40465 {}] regionserver.HRegionServer(1482): Serving as 719ac8e87449,40465,1732607622517, RpcServer on 719ac8e87449/172.17.0.2:40465, sessionid=0x101776bca1c0002 2024-11-26T07:53:44,458 DEBUG [RS:1;719ac8e87449:40465 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-26T07:53:44,458 DEBUG [RS:1;719ac8e87449:40465 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 719ac8e87449,40465,1732607622517 2024-11-26T07:53:44,458 DEBUG [RS:1;719ac8e87449:40465 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '719ac8e87449,40465,1732607622517' 2024-11-26T07:53:44,458 DEBUG [RS:1;719ac8e87449:40465 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-26T07:53:44,460 DEBUG [RS:1;719ac8e87449:40465 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-26T07:53:44,460 INFO [RS:0;719ac8e87449:37023 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-26T07:53:44,460 INFO [RS:0;719ac8e87449:37023 {}] regionserver.HRegionServer(1482): Serving as 719ac8e87449,37023,1732607622257, RpcServer on 719ac8e87449/172.17.0.2:37023, sessionid=0x101776bca1c0001 2024-11-26T07:53:44,460 DEBUG [RS:1;719ac8e87449:40465 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-26T07:53:44,461 DEBUG [RS:1;719ac8e87449:40465 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-26T07:53:44,461 DEBUG [RS:0;719ac8e87449:37023 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-26T07:53:44,461 DEBUG [RS:1;719ac8e87449:40465 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 719ac8e87449,40465,1732607622517 2024-11-26T07:53:44,461 DEBUG [RS:0;719ac8e87449:37023 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 719ac8e87449,37023,1732607622257 2024-11-26T07:53:44,461 DEBUG [RS:1;719ac8e87449:40465 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '719ac8e87449,40465,1732607622517' 2024-11-26T07:53:44,461 DEBUG [RS:0;719ac8e87449:37023 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '719ac8e87449,37023,1732607622257' 2024-11-26T07:53:44,461 DEBUG [RS:1;719ac8e87449:40465 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-26T07:53:44,461 DEBUG [RS:0;719ac8e87449:37023 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-26T07:53:44,462 DEBUG [RS:0;719ac8e87449:37023 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-26T07:53:44,463 DEBUG [RS:1;719ac8e87449:40465 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-26T07:53:44,463 DEBUG [RS:0;719ac8e87449:37023 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-26T07:53:44,463 DEBUG [RS:0;719ac8e87449:37023 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-26T07:53:44,463 DEBUG [RS:0;719ac8e87449:37023 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 719ac8e87449,37023,1732607622257 2024-11-26T07:53:44,463 DEBUG [RS:0;719ac8e87449:37023 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '719ac8e87449,37023,1732607622257' 2024-11-26T07:53:44,463 DEBUG [RS:1;719ac8e87449:40465 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-26T07:53:44,463 DEBUG [RS:0;719ac8e87449:37023 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-26T07:53:44,463 INFO [RS:1;719ac8e87449:40465 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-26T07:53:44,463 INFO [RS:1;719ac8e87449:40465 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-26T07:53:44,464 DEBUG [RS:0;719ac8e87449:37023 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-26T07:53:44,465 DEBUG [RS:0;719ac8e87449:37023 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-26T07:53:44,465 INFO [RS:0;719ac8e87449:37023 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-26T07:53:44,465 INFO [RS:0;719ac8e87449:37023 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-26T07:53:44,554 WARN [719ac8e87449:36589 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-26T07:53:44,569 INFO [RS:1;719ac8e87449:40465 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-26T07:53:44,569 INFO [RS:0;719ac8e87449:37023 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-26T07:53:44,570 INFO [RS:0;719ac8e87449:37023 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=719ac8e87449%2C37023%2C1732607622257, suffix=, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/480cd986-aef2-fe93-7892-18b5e497e91c/293d53df-6485-edb9-9b39-cf743ea8b2f2/WALs/719ac8e87449,37023,1732607622257, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/480cd986-aef2-fe93-7892-18b5e497e91c/293d53df-6485-edb9-9b39-cf743ea8b2f2/oldWALs, maxLogs=55 2024-11-26T07:53:44,570 INFO [RS:1;719ac8e87449:40465 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=719ac8e87449%2C40465%2C1732607622517, suffix=, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/480cd986-aef2-fe93-7892-18b5e497e91c/293d53df-6485-edb9-9b39-cf743ea8b2f2/WALs/719ac8e87449,40465,1732607622517, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/480cd986-aef2-fe93-7892-18b5e497e91c/293d53df-6485-edb9-9b39-cf743ea8b2f2/oldWALs, maxLogs=55 2024-11-26T07:53:44,601 INFO [RS:0;719ac8e87449:37023 {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/480cd986-aef2-fe93-7892-18b5e497e91c/293d53df-6485-edb9-9b39-cf743ea8b2f2/WALs/719ac8e87449,37023,1732607622257/719ac8e87449%2C37023%2C1732607622257.1732607624576 2024-11-26T07:53:44,601 INFO [RS:1;719ac8e87449:40465 {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/480cd986-aef2-fe93-7892-18b5e497e91c/293d53df-6485-edb9-9b39-cf743ea8b2f2/WALs/719ac8e87449,40465,1732607622517/719ac8e87449%2C40465%2C1732607622517.1732607624576 2024-11-26T07:53:44,601 DEBUG [RS:0;719ac8e87449:37023 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-11-26T07:53:44,602 DEBUG [RS:1;719ac8e87449:40465 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-11-26T07:53:44,807 DEBUG [719ac8e87449:36589 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=2, allServersCount=2 2024-11-26T07:53:44,818 DEBUG [719ac8e87449:36589 {}] balancer.BalancerClusterState(204): Hosts are {719ac8e87449=0} racks are {/default-rack=0} 2024-11-26T07:53:44,821 DEBUG [719ac8e87449:36589 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-11-26T07:53:44,821 DEBUG [719ac8e87449:36589 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-11-26T07:53:44,821 DEBUG [719ac8e87449:36589 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-11-26T07:53:44,821 DEBUG [719ac8e87449:36589 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-11-26T07:53:44,821 INFO [719ac8e87449:36589 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-11-26T07:53:44,822 INFO [719ac8e87449:36589 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-11-26T07:53:44,822 DEBUG [719ac8e87449:36589 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-11-26T07:53:44,830 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=719ac8e87449,40465,1732607622517 2024-11-26T07:53:44,836 INFO [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 719ac8e87449,40465,1732607622517, state=OPENING 2024-11-26T07:53:44,867 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2024-11-26T07:53:44,875 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40465-0x101776bca1c0002, quorum=127.0.0.1:53139, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-26T07:53:44,875 DEBUG [pool-14-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37023-0x101776bca1c0001, quorum=127.0.0.1:53139, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-26T07:53:44,875 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36589-0x101776bca1c0000, quorum=127.0.0.1:53139, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-26T07:53:44,877 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-26T07:53:44,877 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-26T07:53:44,878 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-26T07:53:44,879 DEBUG [PEWorker-3 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-26T07:53:44,881 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=719ac8e87449,40465,1732607622517}] 2024-11-26T07:53:45,052 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-26T07:53:45,056 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-4-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42087, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-26T07:53:45,067 INFO [RS_OPEN_META-regionserver/719ac8e87449:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-11-26T07:53:45,067 INFO [RS_OPEN_META-regionserver/719ac8e87449:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-26T07:53:45,068 INFO [RS_OPEN_META-regionserver/719ac8e87449:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-11-26T07:53:45,070 INFO [RS_OPEN_META-regionserver/719ac8e87449:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=719ac8e87449%2C40465%2C1732607622517.meta, suffix=.meta, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/480cd986-aef2-fe93-7892-18b5e497e91c/293d53df-6485-edb9-9b39-cf743ea8b2f2/WALs/719ac8e87449,40465,1732607622517, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/480cd986-aef2-fe93-7892-18b5e497e91c/293d53df-6485-edb9-9b39-cf743ea8b2f2/oldWALs, maxLogs=55 2024-11-26T07:53:45,091 INFO [RS_OPEN_META-regionserver/719ac8e87449:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/480cd986-aef2-fe93-7892-18b5e497e91c/293d53df-6485-edb9-9b39-cf743ea8b2f2/WALs/719ac8e87449,40465,1732607622517/719ac8e87449%2C40465%2C1732607622517.meta.1732607625072.meta 2024-11-26T07:53:45,092 DEBUG [RS_OPEN_META-regionserver/719ac8e87449:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-11-26T07:53:45,093 DEBUG [RS_OPEN_META-regionserver/719ac8e87449:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-11-26T07:53:45,095 DEBUG [RS_OPEN_META-regionserver/719ac8e87449:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-11-26T07:53:45,098 DEBUG [RS_OPEN_META-regionserver/719ac8e87449:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-11-26T07:53:45,104 INFO [RS_OPEN_META-regionserver/719ac8e87449:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-11-26T07:53:45,108 DEBUG [RS_OPEN_META-regionserver/719ac8e87449:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-11-26T07:53:45,109 DEBUG [RS_OPEN_META-regionserver/719ac8e87449:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-26T07:53:45,109 DEBUG [RS_OPEN_META-regionserver/719ac8e87449:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-11-26T07:53:45,109 DEBUG [RS_OPEN_META-regionserver/719ac8e87449:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-11-26T07:53:45,111 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family info of region 1588230740 2024-11-26T07:53:45,112 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-26T07:53:45,112 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T07:53:45,112 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-26T07:53:45,112 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family ns of region 1588230740 2024-11-26T07:53:45,113 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-26T07:53:45,113 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T07:53:45,114 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-26T07:53:45,114 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family rep_barrier of region 1588230740 2024-11-26T07:53:45,115 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-26T07:53:45,115 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T07:53:45,115 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-26T07:53:45,115 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family table of region 1588230740 2024-11-26T07:53:45,116 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-26T07:53:45,116 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T07:53:45,116 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-26T07:53:45,117 DEBUG [RS_OPEN_META-regionserver/719ac8e87449:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-26T07:53:45,118 DEBUG [RS_OPEN_META-regionserver/719ac8e87449:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/480cd986-aef2-fe93-7892-18b5e497e91c/293d53df-6485-edb9-9b39-cf743ea8b2f2/data/hbase/meta/1588230740 2024-11-26T07:53:45,119 DEBUG [RS_OPEN_META-regionserver/719ac8e87449:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/480cd986-aef2-fe93-7892-18b5e497e91c/293d53df-6485-edb9-9b39-cf743ea8b2f2/data/hbase/meta/1588230740 2024-11-26T07:53:45,120 DEBUG [RS_OPEN_META-regionserver/719ac8e87449:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-26T07:53:45,120 DEBUG [RS_OPEN_META-regionserver/719ac8e87449:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-26T07:53:45,120 DEBUG [RS_OPEN_META-regionserver/719ac8e87449:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-26T07:53:45,122 DEBUG [RS_OPEN_META-regionserver/719ac8e87449:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-26T07:53:45,123 INFO [RS_OPEN_META-regionserver/719ac8e87449:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=60615347, jitterRate=-0.09676094353199005}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-26T07:53:45,123 DEBUG [RS_OPEN_META-regionserver/719ac8e87449:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-11-26T07:53:45,126 DEBUG [RS_OPEN_META-regionserver/719ac8e87449:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1732607625110Writing region info on filesystem at 1732607625110Initializing all the Stores at 1732607625110Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732607625110Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732607625110Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732607625111 (+1 ms)Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732607625111Cleaning up temporary data from old regions at 1732607625120 (+9 ms)Running coprocessor post-open hooks at 1732607625123 (+3 ms)Region opened successfully at 1732607625126 (+3 ms) 2024-11-26T07:53:45,131 INFO [RS_OPEN_META-regionserver/719ac8e87449:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1732607625045 2024-11-26T07:53:45,144 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=719ac8e87449,40465,1732607622517 2024-11-26T07:53:45,145 DEBUG [RS_OPEN_META-regionserver/719ac8e87449:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-11-26T07:53:45,145 INFO [RS_OPEN_META-regionserver/719ac8e87449:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-11-26T07:53:45,146 INFO [PEWorker-4 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 719ac8e87449,40465,1732607622517, state=OPEN 2024-11-26T07:53:45,175 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40465-0x101776bca1c0002, quorum=127.0.0.1:53139, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-26T07:53:45,175 DEBUG [pool-14-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37023-0x101776bca1c0001, quorum=127.0.0.1:53139, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-26T07:53:45,176 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-26T07:53:45,176 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-26T07:53:45,175 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36589-0x101776bca1c0000, quorum=127.0.0.1:53139, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-26T07:53:45,176 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-26T07:53:45,177 DEBUG [PEWorker-4 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=719ac8e87449,40465,1732607622517 2024-11-26T07:53:45,180 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2 2024-11-26T07:53:45,181 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=719ac8e87449,40465,1732607622517 in 296 msec 2024-11-26T07:53:45,185 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1 2024-11-26T07:53:45,185 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 797 msec 2024-11-26T07:53:45,186 DEBUG [PEWorker-2 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-26T07:53:45,186 INFO [PEWorker-2 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces 2024-11-26T07:53:45,210 DEBUG [PEWorker-2 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-26T07:53:45,211 DEBUG [PEWorker-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=719ac8e87449,40465,1732607622517, seqNum=-1] 2024-11-26T07:53:45,237 DEBUG [PEWorker-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-26T07:53:45,240 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-4-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51337, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-26T07:53:45,268 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 1.2600 sec 2024-11-26T07:53:45,268 INFO [master/719ac8e87449:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1732607625268, completionTime=-1 2024-11-26T07:53:45,271 INFO [master/719ac8e87449:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=2; waited=0ms, expected min=2 server(s), max=2 server(s), master is running 2024-11-26T07:53:45,271 DEBUG [master/719ac8e87449:0:becomeActiveMaster {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-26T07:53:45,296 INFO [master/719ac8e87449:0:becomeActiveMaster {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-11-26T07:53:45,297 INFO [master/719ac8e87449:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732607685296 2024-11-26T07:53:45,297 INFO [master/719ac8e87449:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732607745297 2024-11-26T07:53:45,297 INFO [master/719ac8e87449:0:becomeActiveMaster {}] assignment.AssignmentManager(1783): Joined the cluster in 25 msec 2024-11-26T07:53:45,300 DEBUG [master/719ac8e87449:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 1588230740 changed from -1.0 to 0.0, refreshing cache 2024-11-26T07:53:45,310 INFO [master/719ac8e87449:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=719ac8e87449,36589,1732607621309-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-26T07:53:45,311 INFO [master/719ac8e87449:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=719ac8e87449,36589,1732607621309-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-26T07:53:45,311 INFO [master/719ac8e87449:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=719ac8e87449,36589,1732607621309-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-26T07:53:45,313 INFO [master/719ac8e87449:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-719ac8e87449:36589, period=300000, unit=MILLISECONDS is enabled. 2024-11-26T07:53:45,313 INFO [master/719ac8e87449:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2024-11-26T07:53:45,314 INFO [master/719ac8e87449:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled. 2024-11-26T07:53:45,325 DEBUG [master/719ac8e87449:0.Chore.1 {}] janitor.CatalogJanitor(180): 2024-11-26T07:53:45,351 INFO [master/719ac8e87449:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 2.605sec 2024-11-26T07:53:45,371 INFO [master/719ac8e87449:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2024-11-26T07:53:45,373 INFO [master/719ac8e87449:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2024-11-26T07:53:45,374 INFO [master/719ac8e87449:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2024-11-26T07:53:45,375 INFO [master/719ac8e87449:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2024-11-26T07:53:45,375 INFO [master/719ac8e87449:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2024-11-26T07:53:45,376 INFO [master/719ac8e87449:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=719ac8e87449,36589,1732607621309-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-26T07:53:45,377 INFO [master/719ac8e87449:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=719ac8e87449,36589,1732607621309-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2024-11-26T07:53:45,401 DEBUG [master/719ac8e87449:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds 2024-11-26T07:53:45,402 INFO [master/719ac8e87449:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2024-11-26T07:53:45,403 INFO [master/719ac8e87449:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=719ac8e87449,36589,1732607621309-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-26T07:53:45,424 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6a8eba43, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T07:53:45,427 DEBUG [Time-limited test {}] nio.NioEventLoop(110): -Dio.netty.noKeySetOptimization: false 2024-11-26T07:53:45,427 DEBUG [Time-limited test {}] nio.NioEventLoop(111): -Dio.netty.selectorAutoRebuildThreshold: 512 2024-11-26T07:53:45,433 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 719ac8e87449,36589,-1 for getting cluster id 2024-11-26T07:53:45,435 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-26T07:53:45,456 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '4d71bb38-f6cf-415f-a628-77b4c190248b' 2024-11-26T07:53:45,459 DEBUG [RPCClient-NioEventLoopGroup-5-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-26T07:53:45,460 DEBUG [RPCClient-NioEventLoopGroup-5-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "4d71bb38-f6cf-415f-a628-77b4c190248b" 2024-11-26T07:53:45,460 DEBUG [RPCClient-NioEventLoopGroup-5-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@749659a2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T07:53:45,461 DEBUG [RPCClient-NioEventLoopGroup-5-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [719ac8e87449,36589,-1] 2024-11-26T07:53:45,464 DEBUG [RPCClient-NioEventLoopGroup-5-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-26T07:53:45,467 DEBUG [RPCClient-NioEventLoopGroup-5-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T07:53:45,469 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40020, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-26T07:53:45,473 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@201cd73f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T07:53:45,474 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-26T07:53:45,484 DEBUG [RPCClient-NioEventLoopGroup-5-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=719ac8e87449,40465,1732607622517, seqNum=-1] 2024-11-26T07:53:45,490 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-26T07:53:45,496 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-4-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36126, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-26T07:53:45,523 INFO [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=719ac8e87449,36589,1732607621309 2024-11-26T07:53:45,527 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-26T07:53:45,532 DEBUG [RPCClient-NioEventLoopGroup-5-2 {}] client.AsyncConnectionImpl(321): The fetched master address is 719ac8e87449,36589,1732607621309 2024-11-26T07:53:45,535 DEBUG [RPCClient-NioEventLoopGroup-5-2 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@11ca9415 2024-11-26T07:53:45,536 DEBUG [RPCClient-NioEventLoopGroup-5-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-26T07:53:45,539 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40024, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-26T07:53:45,546 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36589 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'testBlockEvictionOnRegionMove1732607625523', {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-26T07:53:45,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36589 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=testBlockEvictionOnRegionMove1732607625523 2024-11-26T07:53:45,556 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=testBlockEvictionOnRegionMove1732607625523 execute state=CREATE_TABLE_PRE_OPERATION 2024-11-26T07:53:45,560 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T07:53:45,562 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=testBlockEvictionOnRegionMove1732607625523 execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-26T07:53:45,564 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36589 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "testBlockEvictionOnRegionMove1732607625523" procId is: 4 2024-11-26T07:53:45,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36589 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-26T07:53:45,579 INFO [RegionOpenAndInit-testBlockEvictionOnRegionMove1732607625523-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 419e482dbb0367e2e270d490799809d0, NAME => 'testBlockEvictionOnRegionMove1732607625523,,1732607625541.419e482dbb0367e2e270d490799809d0.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testBlockEvictionOnRegionMove1732607625523', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/480cd986-aef2-fe93-7892-18b5e497e91c/293d53df-6485-edb9-9b39-cf743ea8b2f2 2024-11-26T07:53:45,593 DEBUG [RegionOpenAndInit-testBlockEvictionOnRegionMove1732607625523-pool-0 {}] regionserver.HRegion(898): Instantiated testBlockEvictionOnRegionMove1732607625523,,1732607625541.419e482dbb0367e2e270d490799809d0.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-26T07:53:45,593 DEBUG [RegionOpenAndInit-testBlockEvictionOnRegionMove1732607625523-pool-0 {}] regionserver.HRegion(1722): Closing 419e482dbb0367e2e270d490799809d0, disabling compactions & flushes 2024-11-26T07:53:45,593 INFO [RegionOpenAndInit-testBlockEvictionOnRegionMove1732607625523-pool-0 {}] regionserver.HRegion(1755): Closing region testBlockEvictionOnRegionMove1732607625523,,1732607625541.419e482dbb0367e2e270d490799809d0. 2024-11-26T07:53:45,593 DEBUG [RegionOpenAndInit-testBlockEvictionOnRegionMove1732607625523-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on testBlockEvictionOnRegionMove1732607625523,,1732607625541.419e482dbb0367e2e270d490799809d0. 2024-11-26T07:53:45,593 DEBUG [RegionOpenAndInit-testBlockEvictionOnRegionMove1732607625523-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on testBlockEvictionOnRegionMove1732607625523,,1732607625541.419e482dbb0367e2e270d490799809d0. after waiting 0 ms 2024-11-26T07:53:45,593 DEBUG [RegionOpenAndInit-testBlockEvictionOnRegionMove1732607625523-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region testBlockEvictionOnRegionMove1732607625523,,1732607625541.419e482dbb0367e2e270d490799809d0. 2024-11-26T07:53:45,593 INFO [RegionOpenAndInit-testBlockEvictionOnRegionMove1732607625523-pool-0 {}] regionserver.HRegion(1973): Closed testBlockEvictionOnRegionMove1732607625523,,1732607625541.419e482dbb0367e2e270d490799809d0. 2024-11-26T07:53:45,593 DEBUG [RegionOpenAndInit-testBlockEvictionOnRegionMove1732607625523-pool-0 {}] regionserver.HRegion(1676): Region close journal for 419e482dbb0367e2e270d490799809d0: Waiting for close lock at 1732607625593Disabling compacts and flushes for region at 1732607625593Disabling writes for close at 1732607625593Writing region close event to WAL at 1732607625593Closed at 1732607625593 2024-11-26T07:53:45,597 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=testBlockEvictionOnRegionMove1732607625523 execute state=CREATE_TABLE_ADD_TO_META 2024-11-26T07:53:45,603 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testBlockEvictionOnRegionMove1732607625523,,1732607625541.419e482dbb0367e2e270d490799809d0.","families":{"info":[{"qualifier":"regioninfo","vlen":76,"tag":[],"timestamp":"1732607625597"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732607625597"}]},"ts":"1732607625597"} 2024-11-26T07:53:45,608 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-26T07:53:45,610 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=testBlockEvictionOnRegionMove1732607625523 execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-26T07:53:45,613 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testBlockEvictionOnRegionMove1732607625523","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732607625610"}]},"ts":"1732607625610"} 2024-11-26T07:53:45,617 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=testBlockEvictionOnRegionMove1732607625523, state=ENABLING in hbase:meta 2024-11-26T07:53:45,618 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(204): Hosts are {719ac8e87449=0} racks are {/default-rack=0} 2024-11-26T07:53:45,618 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-11-26T07:53:45,618 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-11-26T07:53:45,618 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-11-26T07:53:45,618 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-11-26T07:53:45,619 INFO [PEWorker-3 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-11-26T07:53:45,619 INFO [PEWorker-3 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-11-26T07:53:45,619 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-11-26T07:53:45,620 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1732607625523, region=419e482dbb0367e2e270d490799809d0, ASSIGN}] 2024-11-26T07:53:45,622 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1732607625523, region=419e482dbb0367e2e270d490799809d0, ASSIGN 2024-11-26T07:53:45,626 INFO [PEWorker-5 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1732607625523, region=419e482dbb0367e2e270d490799809d0, ASSIGN; state=OFFLINE, location=719ac8e87449,40465,1732607622517; forceNewPlan=false, retain=false 2024-11-26T07:53:45,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36589 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-26T07:53:45,779 INFO [719ac8e87449:36589 {}] balancer.BaseLoadBalancer(388): Reassigned 1 regions. 1 retained the pre-restart assignment. 2024-11-26T07:53:45,780 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=419e482dbb0367e2e270d490799809d0, regionState=OPENING, regionLocation=719ac8e87449,40465,1732607622517 2024-11-26T07:53:45,785 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-4-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1732607625523, region=419e482dbb0367e2e270d490799809d0, ASSIGN because future has completed 2024-11-26T07:53:45,786 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 419e482dbb0367e2e270d490799809d0, server=719ac8e87449,40465,1732607622517}] 2024-11-26T07:53:45,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36589 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-26T07:53:45,942 INFO [RS_OPEN_REGION-regionserver/719ac8e87449:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(132): Open testBlockEvictionOnRegionMove1732607625523,,1732607625541.419e482dbb0367e2e270d490799809d0. 2024-11-26T07:53:45,943 DEBUG [RS_OPEN_REGION-regionserver/719ac8e87449:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7752): Opening region: {ENCODED => 419e482dbb0367e2e270d490799809d0, NAME => 'testBlockEvictionOnRegionMove1732607625523,,1732607625541.419e482dbb0367e2e270d490799809d0.', STARTKEY => '', ENDKEY => ''} 2024-11-26T07:53:45,944 DEBUG [RS_OPEN_REGION-regionserver/719ac8e87449:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testBlockEvictionOnRegionMove1732607625523 419e482dbb0367e2e270d490799809d0 2024-11-26T07:53:45,944 DEBUG [RS_OPEN_REGION-regionserver/719ac8e87449:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(898): Instantiated testBlockEvictionOnRegionMove1732607625523,,1732607625541.419e482dbb0367e2e270d490799809d0.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-26T07:53:45,944 DEBUG [RS_OPEN_REGION-regionserver/719ac8e87449:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7794): checking encryption for 419e482dbb0367e2e270d490799809d0 2024-11-26T07:53:45,944 DEBUG [RS_OPEN_REGION-regionserver/719ac8e87449:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7797): checking classloading for 419e482dbb0367e2e270d490799809d0 2024-11-26T07:53:45,945 INFO [StoreOpener-419e482dbb0367e2e270d490799809d0-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family family of region 419e482dbb0367e2e270d490799809d0 2024-11-26T07:53:45,950 INFO [StoreOpener-419e482dbb0367e2e270d490799809d0-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 419e482dbb0367e2e270d490799809d0 columnFamilyName family 2024-11-26T07:53:45,951 DEBUG [StoreOpener-419e482dbb0367e2e270d490799809d0-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T07:53:45,951 INFO [StoreOpener-419e482dbb0367e2e270d490799809d0-1 {}] regionserver.HStore(327): Store=419e482dbb0367e2e270d490799809d0/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-26T07:53:45,954 DEBUG [RS_OPEN_REGION-regionserver/719ac8e87449:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1038): replaying wal for 419e482dbb0367e2e270d490799809d0 2024-11-26T07:53:45,956 DEBUG [RS_OPEN_REGION-regionserver/719ac8e87449:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/480cd986-aef2-fe93-7892-18b5e497e91c/293d53df-6485-edb9-9b39-cf743ea8b2f2/data/default/testBlockEvictionOnRegionMove1732607625523/419e482dbb0367e2e270d490799809d0 2024-11-26T07:53:45,956 DEBUG [RS_OPEN_REGION-regionserver/719ac8e87449:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/480cd986-aef2-fe93-7892-18b5e497e91c/293d53df-6485-edb9-9b39-cf743ea8b2f2/data/default/testBlockEvictionOnRegionMove1732607625523/419e482dbb0367e2e270d490799809d0 2024-11-26T07:53:45,956 DEBUG [RS_OPEN_REGION-regionserver/719ac8e87449:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1048): stopping wal replay for 419e482dbb0367e2e270d490799809d0 2024-11-26T07:53:45,956 DEBUG [RS_OPEN_REGION-regionserver/719ac8e87449:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1060): Cleaning up temporary data for 419e482dbb0367e2e270d490799809d0 2024-11-26T07:53:45,957 DEBUG [RS_OPEN_REGION-regionserver/719ac8e87449:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1093): writing seq id for 419e482dbb0367e2e270d490799809d0 2024-11-26T07:53:45,963 DEBUG [RS_OPEN_REGION-regionserver/719ac8e87449:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/480cd986-aef2-fe93-7892-18b5e497e91c/293d53df-6485-edb9-9b39-cf743ea8b2f2/data/default/testBlockEvictionOnRegionMove1732607625523/419e482dbb0367e2e270d490799809d0/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-26T07:53:45,964 INFO [RS_OPEN_REGION-regionserver/719ac8e87449:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1114): Opened 419e482dbb0367e2e270d490799809d0; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=59921803, jitterRate=-0.10709555447101593}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-26T07:53:45,964 DEBUG [RS_OPEN_REGION-regionserver/719ac8e87449:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 419e482dbb0367e2e270d490799809d0 2024-11-26T07:53:45,966 DEBUG [RS_OPEN_REGION-regionserver/719ac8e87449:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1006): Region open journal for 419e482dbb0367e2e270d490799809d0: Running coprocessor pre-open hook at 1732607625944Writing region info on filesystem at 1732607625944Initializing all the Stores at 1732607625945 (+1 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732607625945Cleaning up temporary data from old regions at 1732607625956 (+11 ms)Running coprocessor post-open hooks at 1732607625964 (+8 ms)Region opened successfully at 1732607625966 (+2 ms) 2024-11-26T07:53:45,967 INFO [RS_OPEN_REGION-regionserver/719ac8e87449:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2236): Post open deploy tasks for testBlockEvictionOnRegionMove1732607625523,,1732607625541.419e482dbb0367e2e270d490799809d0., pid=6, masterSystemTime=1732607625939 2024-11-26T07:53:45,973 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=419e482dbb0367e2e270d490799809d0, regionState=OPEN, openSeqNum=2, regionLocation=719ac8e87449,40465,1732607622517 2024-11-26T07:53:45,975 DEBUG [RS_OPEN_REGION-regionserver/719ac8e87449:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2266): Finished post open deploy task for testBlockEvictionOnRegionMove1732607625523,,1732607625541.419e482dbb0367e2e270d490799809d0. 2024-11-26T07:53:45,975 INFO [RS_OPEN_REGION-regionserver/719ac8e87449:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(153): Opened testBlockEvictionOnRegionMove1732607625523,,1732607625541.419e482dbb0367e2e270d490799809d0. 2024-11-26T07:53:45,977 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-4-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 419e482dbb0367e2e270d490799809d0, server=719ac8e87449,40465,1732607622517 because future has completed 2024-11-26T07:53:45,981 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=6, resume processing ppid=5 2024-11-26T07:53:45,981 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure 419e482dbb0367e2e270d490799809d0, server=719ac8e87449,40465,1732607622517 in 193 msec 2024-11-26T07:53:45,983 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=5, resume processing ppid=4 2024-11-26T07:53:45,984 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1732607625523, region=419e482dbb0367e2e270d490799809d0, ASSIGN in 361 msec 2024-11-26T07:53:45,984 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=testBlockEvictionOnRegionMove1732607625523 execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-26T07:53:45,985 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testBlockEvictionOnRegionMove1732607625523","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732607625984"}]},"ts":"1732607625984"} 2024-11-26T07:53:45,987 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=testBlockEvictionOnRegionMove1732607625523, state=ENABLED in hbase:meta 2024-11-26T07:53:45,989 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=testBlockEvictionOnRegionMove1732607625523 execute state=CREATE_TABLE_POST_OPERATION 2024-11-26T07:53:45,991 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=testBlockEvictionOnRegionMove1732607625523 in 440 msec 2024-11-26T07:53:46,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36589 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-26T07:53:46,199 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table testBlockEvictionOnRegionMove1732607625523 get assigned. Timeout = 60000ms 2024-11-26T07:53:46,199 INFO [RPCClient-NioEventLoopGroup-5-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:testBlockEvictionOnRegionMove1732607625523 completed 2024-11-26T07:53:46,202 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-26T07:53:46,210 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table testBlockEvictionOnRegionMove1732607625523 assigned to meta. Checking AM states. 2024-11-26T07:53:46,210 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-26T07:53:46,211 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table testBlockEvictionOnRegionMove1732607625523 assigned. 2024-11-26T07:53:46,220 DEBUG [RPCClient-NioEventLoopGroup-5-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testBlockEvictionOnRegionMove1732607625523', row='row1', locateType=CURRENT is [region=testBlockEvictionOnRegionMove1732607625523,,1732607625541.419e482dbb0367e2e270d490799809d0., hostname=719ac8e87449,40465,1732607622517, seqNum=2] 2024-11-26T07:53:46,235 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 419e482dbb0367e2e270d490799809d0 1/1 column families, dataSize=78 B heapSize=480 B 2024-11-26T07:53:46,260 DEBUG [Time-limited test {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on e67d40e19a6d427cadef5c414992c907 2024-11-26T07:53:46,298 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/480cd986-aef2-fe93-7892-18b5e497e91c/293d53df-6485-edb9-9b39-cf743ea8b2f2/data/default/testBlockEvictionOnRegionMove1732607625523/419e482dbb0367e2e270d490799809d0/.tmp/family/e67d40e19a6d427cadef5c414992c907 is 43, key is row1/family:qf1/1/Put/seqid=0 2024-11-26T07:53:46,303 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=78 B at sequenceid=6 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/480cd986-aef2-fe93-7892-18b5e497e91c/293d53df-6485-edb9-9b39-cf743ea8b2f2/data/default/testBlockEvictionOnRegionMove1732607625523/419e482dbb0367e2e270d490799809d0/.tmp/family/e67d40e19a6d427cadef5c414992c907 2024-11-26T07:53:46,342 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/480cd986-aef2-fe93-7892-18b5e497e91c/293d53df-6485-edb9-9b39-cf743ea8b2f2/data/default/testBlockEvictionOnRegionMove1732607625523/419e482dbb0367e2e270d490799809d0/.tmp/family/e67d40e19a6d427cadef5c414992c907 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/480cd986-aef2-fe93-7892-18b5e497e91c/293d53df-6485-edb9-9b39-cf743ea8b2f2/data/default/testBlockEvictionOnRegionMove1732607625523/419e482dbb0367e2e270d490799809d0/family/e67d40e19a6d427cadef5c414992c907 2024-11-26T07:53:46,343 DEBUG [Time-limited test {}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/480cd986-aef2-fe93-7892-18b5e497e91c/293d53df-6485-edb9-9b39-cf743ea8b2f2/data/default/testBlockEvictionOnRegionMove1732607625523/419e482dbb0367e2e270d490799809d0/family/e67d40e19a6d427cadef5c414992c907, delay=913 ms 2024-11-26T07:53:46,345 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/480cd986-aef2-fe93-7892-18b5e497e91c/293d53df-6485-edb9-9b39-cf743ea8b2f2/data/default/testBlockEvictionOnRegionMove1732607625523/419e482dbb0367e2e270d490799809d0/family/e67d40e19a6d427cadef5c414992c907, entries=2, sequenceid=6, filesize=4.9 K 2024-11-26T07:53:46,351 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~78 B/78, heapSize ~464 B/464, currentSize=0 B/0 for 419e482dbb0367e2e270d490799809d0 in 115ms, sequenceid=6, compaction requested=false 2024-11-26T07:53:46,352 DEBUG [Time-limited test {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'testBlockEvictionOnRegionMove1732607625523' 2024-11-26T07:53:46,354 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for 419e482dbb0367e2e270d490799809d0: 2024-11-26T07:53:47,259 DEBUG [hfile-prefetch-1732607626343 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key e67d40e19a6d427cadef5c414992c907_0. 2024-11-26T07:53:47,264 DEBUG [hfile-prefetch-1732607626343 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key e67d40e19a6d427cadef5c414992c907_117. 2024-11-26T07:53:47,265 DEBUG [hfile-prefetch-1732607626343 {}] bucket.BucketCache(2281): Notifying caching completed for file file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/480cd986-aef2-fe93-7892-18b5e497e91c/293d53df-6485-edb9-9b39-cf743ea8b2f2/data/default/testBlockEvictionOnRegionMove1732607625523/419e482dbb0367e2e270d490799809d0/family/e67d40e19a6d427cadef5c414992c907, with total blocks 2, and data blocks 1 2024-11-26T07:53:47,265 DEBUG [hfile-prefetch-1732607626343 {}] bucket.BucketCache(2285): iterating over 0 entries in the backing map 2024-11-26T07:53:47,265 DEBUG [hfile-prefetch-1732607626343 {}] bucket.BucketCache(2308): Prefetch executor completed for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/480cd986-aef2-fe93-7892-18b5e497e91c/293d53df-6485-edb9-9b39-cf743ea8b2f2/data/default/testBlockEvictionOnRegionMove1732607625523/419e482dbb0367e2e270d490799809d0/family/e67d40e19a6d427cadef5c414992c907, but only 0 data blocks were cached. Total data blocks for file: 1. Checking for blocks pending cache in cache writer queue. 2024-11-26T07:53:47,265 DEBUG [hfile-prefetch-1732607626343 {}] bucket.BucketCache(2318): There are still blocks pending caching for file file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/480cd986-aef2-fe93-7892-18b5e497e91c/293d53df-6485-edb9-9b39-cf743ea8b2f2/data/default/testBlockEvictionOnRegionMove1732607625523/419e482dbb0367e2e270d490799809d0/family/e67d40e19a6d427cadef5c414992c907. Will sleep 100ms and try the verification again. 2024-11-26T07:53:47,329 DEBUG [bucket-cache-persister {}] bucket.BucketCachePersister(47): Cache is inconsistent, persisting to disk 2024-11-26T07:53:47,329 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1487): Thread bucket-cache-persister started persisting bucket cache to file 2024-11-26T07:53:47,330 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1494): Persist in new chunked persistence format. 2024-11-26T07:53:47,330 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1673): persistToFile: before persisting backing map size: 0, fullycachedFiles size: 0, chunkSize: 10000 2024-11-26T07:53:47,347 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1680): persistToFile: after persisting backing map size: 0, fullycachedFiles size: 0 2024-11-26T07:53:47,348 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1498): PersistToFile: after persisting backing map size: 0, fullycachedFiles size: 0, file name: bucket.persistence1732607627329 2024-11-26T07:53:47,348 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1509): Thread bucket-cache-persister finished persisting bucket cache to file, renaming 2024-11-26T07:53:47,366 DEBUG [hfile-prefetch-1732607626343 {}] bucket.BucketCache(2281): Notifying caching completed for file file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/480cd986-aef2-fe93-7892-18b5e497e91c/293d53df-6485-edb9-9b39-cf743ea8b2f2/data/default/testBlockEvictionOnRegionMove1732607625523/419e482dbb0367e2e270d490799809d0/family/e67d40e19a6d427cadef5c414992c907, with total blocks 2, and data blocks 1 2024-11-26T07:53:47,366 DEBUG [hfile-prefetch-1732607626343 {}] bucket.BucketCache(2285): iterating over 1 entries in the backing map 2024-11-26T07:53:47,366 DEBUG [hfile-prefetch-1732607626343 {}] bucket.BucketCache(2293): found block for file e67d40e19a6d427cadef5c414992c907 in the backing map. Acquiring read lock for offset 0 2024-11-26T07:53:47,366 DEBUG [hfile-prefetch-1732607626343 {}] bucket.BucketCache(2305): File file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/480cd986-aef2-fe93-7892-18b5e497e91c/293d53df-6485-edb9-9b39-cf743ea8b2f2/data/default/testBlockEvictionOnRegionMove1732607625523/419e482dbb0367e2e270d490799809d0/family/e67d40e19a6d427cadef5c414992c907 has now been fully cached. 2024-11-26T07:53:47,366 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36589 {}] master.HMaster(2410): Client=jenkins//172.17.0.2 move hri=419e482dbb0367e2e270d490799809d0, source=719ac8e87449,40465,1732607622517, destination=719ac8e87449,37023,1732607622257, warming up region on 719ac8e87449,37023,1732607622257 2024-11-26T07:53:47,367 DEBUG [hfile-prefetch-1732607626343 {}] hfile.PrefetchExecutor(121): Prefetch completed for e67d40e19a6d427cadef5c414992c907 2024-11-26T07:53:47,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36589 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-26T07:53:47,372 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36589 {}] master.HMaster(2414): Client=jenkins//172.17.0.2 move hri=419e482dbb0367e2e270d490799809d0, source=719ac8e87449,40465,1732607622517, destination=719ac8e87449,37023,1732607622257, running balancer 2024-11-26T07:53:47,372 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43901, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-26T07:53:47,374 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37023 {}] regionserver.RSRpcServices(2066): Warmup testBlockEvictionOnRegionMove1732607625523,,1732607625541.419e482dbb0367e2e270d490799809d0. 2024-11-26T07:53:47,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37023 {}] regionserver.HRegion(7855): Warmup {ENCODED => 419e482dbb0367e2e270d490799809d0, NAME => 'testBlockEvictionOnRegionMove1732607625523,,1732607625541.419e482dbb0367e2e270d490799809d0.', STARTKEY => '', ENDKEY => ''} 2024-11-26T07:53:47,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36589 {}] procedure2.ProcedureExecutor(1139): Stored pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1732607625523, region=419e482dbb0367e2e270d490799809d0, REOPEN/MOVE 2024-11-26T07:53:47,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37023 {}] regionserver.HRegion(898): Instantiated testBlockEvictionOnRegionMove1732607625523,,1732607625541.419e482dbb0367e2e270d490799809d0.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-26T07:53:47,375 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1732607625523, region=419e482dbb0367e2e270d490799809d0, REOPEN/MOVE 2024-11-26T07:53:47,377 INFO [StoreOpener-419e482dbb0367e2e270d490799809d0-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family family of region 419e482dbb0367e2e270d490799809d0 2024-11-26T07:53:47,377 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=419e482dbb0367e2e270d490799809d0, regionState=CLOSING, regionLocation=719ac8e87449,40465,1732607622517 2024-11-26T07:53:47,382 INFO [StoreOpener-419e482dbb0367e2e270d490799809d0-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 419e482dbb0367e2e270d490799809d0 columnFamilyName family 2024-11-26T07:53:47,382 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-4-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1732607625523, region=419e482dbb0367e2e270d490799809d0, REOPEN/MOVE because future has completed 2024-11-26T07:53:47,382 DEBUG [StoreOpener-419e482dbb0367e2e270d490799809d0-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T07:53:47,383 DEBUG [PEWorker-3 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: true 2024-11-26T07:53:47,383 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=8, ppid=7, state=RUNNABLE, hasLock=false; CloseRegionProcedure 419e482dbb0367e2e270d490799809d0, server=719ac8e87449,40465,1732607622517}] 2024-11-26T07:53:47,391 DEBUG [StoreOpener-419e482dbb0367e2e270d490799809d0-1 {}] regionserver.StoreEngine(278): loaded file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/480cd986-aef2-fe93-7892-18b5e497e91c/293d53df-6485-edb9-9b39-cf743ea8b2f2/data/default/testBlockEvictionOnRegionMove1732607625523/419e482dbb0367e2e270d490799809d0/family/e67d40e19a6d427cadef5c414992c907 2024-11-26T07:53:47,391 INFO [StoreOpener-419e482dbb0367e2e270d490799809d0-1 {}] regionserver.HStore(327): Store=419e482dbb0367e2e270d490799809d0/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-26T07:53:47,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37023 {}] regionserver.HRegion(1722): Closing 419e482dbb0367e2e270d490799809d0, disabling compactions & flushes 2024-11-26T07:53:47,392 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37023 {}] regionserver.HRegion(1755): Closing region testBlockEvictionOnRegionMove1732607625523,,1732607625541.419e482dbb0367e2e270d490799809d0. 2024-11-26T07:53:47,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37023 {}] regionserver.HRegion(1776): Time limited wait for close lock on testBlockEvictionOnRegionMove1732607625523,,1732607625541.419e482dbb0367e2e270d490799809d0. 2024-11-26T07:53:47,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37023 {}] regionserver.HRegion(1843): Acquired close lock on testBlockEvictionOnRegionMove1732607625523,,1732607625541.419e482dbb0367e2e270d490799809d0. after waiting 0 ms 2024-11-26T07:53:47,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37023 {}] regionserver.HRegion(1853): Updates disabled for region testBlockEvictionOnRegionMove1732607625523,,1732607625541.419e482dbb0367e2e270d490799809d0. 2024-11-26T07:53:47,393 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37023 {}] regionserver.HRegion(1973): Closed testBlockEvictionOnRegionMove1732607625523,,1732607625541.419e482dbb0367e2e270d490799809d0. 2024-11-26T07:53:47,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37023 {}] regionserver.HRegion(1676): Region close journal for 419e482dbb0367e2e270d490799809d0: Waiting for close lock at 1732607627391Disabling compacts and flushes for region at 1732607627391Disabling writes for close at 1732607627392 (+1 ms)Writing region close event to WAL at 1732607627393 (+1 ms)Closed at 1732607627393 2024-11-26T07:53:47,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36589 {}] procedure.ProcedureSyncWait(219): waitFor pid=7 2024-11-26T07:53:47,541 INFO [RS_CLOSE_REGION-regionserver/719ac8e87449:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] handler.UnassignRegionHandler(122): Close 419e482dbb0367e2e270d490799809d0 2024-11-26T07:53:47,542 DEBUG [RS_CLOSE_REGION-regionserver/719ac8e87449:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: true 2024-11-26T07:53:47,542 DEBUG [RS_CLOSE_REGION-regionserver/719ac8e87449:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1722): Closing 419e482dbb0367e2e270d490799809d0, disabling compactions & flushes 2024-11-26T07:53:47,542 INFO [RS_CLOSE_REGION-regionserver/719ac8e87449:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1755): Closing region testBlockEvictionOnRegionMove1732607625523,,1732607625541.419e482dbb0367e2e270d490799809d0. 2024-11-26T07:53:47,542 DEBUG [RS_CLOSE_REGION-regionserver/719ac8e87449:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1776): Time limited wait for close lock on testBlockEvictionOnRegionMove1732607625523,,1732607625541.419e482dbb0367e2e270d490799809d0. 2024-11-26T07:53:47,543 DEBUG [RS_CLOSE_REGION-regionserver/719ac8e87449:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1843): Acquired close lock on testBlockEvictionOnRegionMove1732607625523,,1732607625541.419e482dbb0367e2e270d490799809d0. after waiting 1 ms 2024-11-26T07:53:47,543 DEBUG [RS_CLOSE_REGION-regionserver/719ac8e87449:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1853): Updates disabled for region testBlockEvictionOnRegionMove1732607625523,,1732607625541.419e482dbb0367e2e270d490799809d0. 2024-11-26T07:53:47,544 DEBUG [StoreFileCloser-419e482dbb0367e2e270d490799809d0-family-1 {}] bucket.BucketCache(768): Removing file e67d40e19a6d427cadef5c414992c907 for region 419e482dbb0367e2e270d490799809d0 2024-11-26T07:53:47,545 DEBUG [StoreFileCloser-419e482dbb0367e2e270d490799809d0-family-1 {}] bucket.BucketCache(1863): found 1 blocks for file e67d40e19a6d427cadef5c414992c907, starting offset: 0, end offset: 9223372036854775807 2024-11-26T07:53:47,546 DEBUG [StoreFileCloser-419e482dbb0367e2e270d490799809d0-family-1 {}] bucket.BucketCache(856): removed key e67d40e19a6d427cadef5c414992c907_0 from back map with offset lock 2094080 in the evict process 2024-11-26T07:53:47,548 DEBUG [bucket-cache-persister {}] bucket.BucketCachePersister(47): Cache is inconsistent, persisting to disk 2024-11-26T07:53:47,548 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1487): Thread bucket-cache-persister started persisting bucket cache to file 2024-11-26T07:53:47,548 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1494): Persist in new chunked persistence format. 2024-11-26T07:53:47,548 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1673): persistToFile: before persisting backing map size: 0, fullycachedFiles size: 0, chunkSize: 10000 2024-11-26T07:53:47,552 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1680): persistToFile: after persisting backing map size: 0, fullycachedFiles size: 0 2024-11-26T07:53:47,552 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1498): PersistToFile: after persisting backing map size: 0, fullycachedFiles size: 0, file name: bucket.persistence1732607627548 2024-11-26T07:53:47,552 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1509): Thread bucket-cache-persister finished persisting bucket cache to file, renaming 2024-11-26T07:53:47,557 DEBUG [RS_CLOSE_REGION-regionserver/719ac8e87449:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/480cd986-aef2-fe93-7892-18b5e497e91c/293d53df-6485-edb9-9b39-cf743ea8b2f2/data/default/testBlockEvictionOnRegionMove1732607625523/419e482dbb0367e2e270d490799809d0/recovered.edits/9.seqid, newMaxSeqId=9, maxSeqId=1 2024-11-26T07:53:47,558 INFO [RS_CLOSE_REGION-regionserver/719ac8e87449:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1973): Closed testBlockEvictionOnRegionMove1732607625523,,1732607625541.419e482dbb0367e2e270d490799809d0. 2024-11-26T07:53:47,558 DEBUG [RS_CLOSE_REGION-regionserver/719ac8e87449:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1676): Region close journal for 419e482dbb0367e2e270d490799809d0: Waiting for close lock at 1732607627542Running coprocessor pre-close hooks at 1732607627542Disabling compacts and flushes for region at 1732607627542Disabling writes for close at 1732607627543 (+1 ms)Writing region close event to WAL at 1732607627547 (+4 ms)Running coprocessor post-close hooks at 1732607627557 (+10 ms)Closed at 1732607627558 (+1 ms) 2024-11-26T07:53:47,559 INFO [RS_CLOSE_REGION-regionserver/719ac8e87449:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegionServer(3302): Adding 419e482dbb0367e2e270d490799809d0 move to 719ac8e87449,37023,1732607622257 record at close sequenceid=6 2024-11-26T07:53:47,562 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=419e482dbb0367e2e270d490799809d0, regionState=CLOSED 2024-11-26T07:53:47,562 INFO [RS_CLOSE_REGION-regionserver/719ac8e87449:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] handler.UnassignRegionHandler(157): Closed 419e482dbb0367e2e270d490799809d0 2024-11-26T07:53:47,566 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-4-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=8, ppid=7, state=RUNNABLE, hasLock=false; CloseRegionProcedure 419e482dbb0367e2e270d490799809d0, server=719ac8e87449,40465,1732607622517 because future has completed 2024-11-26T07:53:47,570 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=8, resume processing ppid=7 2024-11-26T07:53:47,570 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=8, ppid=7, state=SUCCESS, hasLock=false; CloseRegionProcedure 419e482dbb0367e2e270d490799809d0, server=719ac8e87449,40465,1732607622517 in 184 msec 2024-11-26T07:53:47,570 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1732607625523, region=419e482dbb0367e2e270d490799809d0, REOPEN/MOVE; state=CLOSED, location=719ac8e87449,37023,1732607622257; forceNewPlan=false, retain=false 2024-11-26T07:53:47,721 INFO [719ac8e87449:36589 {}] balancer.BaseLoadBalancer(388): Reassigned 1 regions. 1 retained the pre-restart assignment. 2024-11-26T07:53:47,721 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=419e482dbb0367e2e270d490799809d0, regionState=OPENING, regionLocation=719ac8e87449,37023,1732607622257 2024-11-26T07:53:47,725 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-4-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1732607625523, region=419e482dbb0367e2e270d490799809d0, REOPEN/MOVE because future has completed 2024-11-26T07:53:47,725 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=9, ppid=7, state=RUNNABLE, hasLock=false; OpenRegionProcedure 419e482dbb0367e2e270d490799809d0, server=719ac8e87449,37023,1732607622257}] 2024-11-26T07:53:47,880 INFO [RS_OPEN_REGION-regionserver/719ac8e87449:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] handler.AssignRegionHandler(132): Open testBlockEvictionOnRegionMove1732607625523,,1732607625541.419e482dbb0367e2e270d490799809d0. 2024-11-26T07:53:47,880 DEBUG [RS_OPEN_REGION-regionserver/719ac8e87449:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(7752): Opening region: {ENCODED => 419e482dbb0367e2e270d490799809d0, NAME => 'testBlockEvictionOnRegionMove1732607625523,,1732607625541.419e482dbb0367e2e270d490799809d0.', STARTKEY => '', ENDKEY => ''} 2024-11-26T07:53:47,881 DEBUG [RS_OPEN_REGION-regionserver/719ac8e87449:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testBlockEvictionOnRegionMove1732607625523 419e482dbb0367e2e270d490799809d0 2024-11-26T07:53:47,881 DEBUG [RS_OPEN_REGION-regionserver/719ac8e87449:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(898): Instantiated testBlockEvictionOnRegionMove1732607625523,,1732607625541.419e482dbb0367e2e270d490799809d0.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-26T07:53:47,881 DEBUG [RS_OPEN_REGION-regionserver/719ac8e87449:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(7794): checking encryption for 419e482dbb0367e2e270d490799809d0 2024-11-26T07:53:47,881 DEBUG [RS_OPEN_REGION-regionserver/719ac8e87449:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(7797): checking classloading for 419e482dbb0367e2e270d490799809d0 2024-11-26T07:53:47,882 INFO [StoreOpener-419e482dbb0367e2e270d490799809d0-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family family of region 419e482dbb0367e2e270d490799809d0 2024-11-26T07:53:47,883 INFO [StoreOpener-419e482dbb0367e2e270d490799809d0-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 419e482dbb0367e2e270d490799809d0 columnFamilyName family 2024-11-26T07:53:47,883 DEBUG [StoreOpener-419e482dbb0367e2e270d490799809d0-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T07:53:47,885 DEBUG [StoreFileOpener-419e482dbb0367e2e270d490799809d0-family-1 {}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/480cd986-aef2-fe93-7892-18b5e497e91c/293d53df-6485-edb9-9b39-cf743ea8b2f2/data/default/testBlockEvictionOnRegionMove1732607625523/419e482dbb0367e2e270d490799809d0/family/e67d40e19a6d427cadef5c414992c907, delay=996 ms 2024-11-26T07:53:47,887 DEBUG [StoreOpener-419e482dbb0367e2e270d490799809d0-1 {}] regionserver.StoreEngine(278): loaded file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/480cd986-aef2-fe93-7892-18b5e497e91c/293d53df-6485-edb9-9b39-cf743ea8b2f2/data/default/testBlockEvictionOnRegionMove1732607625523/419e482dbb0367e2e270d490799809d0/family/e67d40e19a6d427cadef5c414992c907 2024-11-26T07:53:47,887 INFO [StoreOpener-419e482dbb0367e2e270d490799809d0-1 {}] regionserver.HStore(327): Store=419e482dbb0367e2e270d490799809d0/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-26T07:53:47,887 DEBUG [RS_OPEN_REGION-regionserver/719ac8e87449:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1038): replaying wal for 419e482dbb0367e2e270d490799809d0 2024-11-26T07:53:47,888 DEBUG [RS_OPEN_REGION-regionserver/719ac8e87449:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/480cd986-aef2-fe93-7892-18b5e497e91c/293d53df-6485-edb9-9b39-cf743ea8b2f2/data/default/testBlockEvictionOnRegionMove1732607625523/419e482dbb0367e2e270d490799809d0 2024-11-26T07:53:47,888 DEBUG [RS_OPEN_REGION-regionserver/719ac8e87449:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/480cd986-aef2-fe93-7892-18b5e497e91c/293d53df-6485-edb9-9b39-cf743ea8b2f2/data/default/testBlockEvictionOnRegionMove1732607625523/419e482dbb0367e2e270d490799809d0 2024-11-26T07:53:47,888 DEBUG [RS_OPEN_REGION-regionserver/719ac8e87449:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1048): stopping wal replay for 419e482dbb0367e2e270d490799809d0 2024-11-26T07:53:47,889 DEBUG [RS_OPEN_REGION-regionserver/719ac8e87449:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1060): Cleaning up temporary data for 419e482dbb0367e2e270d490799809d0 2024-11-26T07:53:47,892 DEBUG [RS_OPEN_REGION-regionserver/719ac8e87449:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1093): writing seq id for 419e482dbb0367e2e270d490799809d0 2024-11-26T07:53:47,893 INFO [RS_OPEN_REGION-regionserver/719ac8e87449:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1114): Opened 419e482dbb0367e2e270d490799809d0; next sequenceid=10; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=74479501, jitterRate=0.10983105003833771}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-26T07:53:47,893 DEBUG [RS_OPEN_REGION-regionserver/719ac8e87449:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 419e482dbb0367e2e270d490799809d0 2024-11-26T07:53:47,894 DEBUG [RS_OPEN_REGION-regionserver/719ac8e87449:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1006): Region open journal for 419e482dbb0367e2e270d490799809d0: Running coprocessor pre-open hook at 1732607627881Writing region info on filesystem at 1732607627881Initializing all the Stores at 1732607627881Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732607627882 (+1 ms)Cleaning up temporary data from old regions at 1732607627889 (+7 ms)Running coprocessor post-open hooks at 1732607627893 (+4 ms)Region opened successfully at 1732607627894 (+1 ms) 2024-11-26T07:53:47,895 INFO [RS_OPEN_REGION-regionserver/719ac8e87449:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegionServer(2236): Post open deploy tasks for testBlockEvictionOnRegionMove1732607625523,,1732607625541.419e482dbb0367e2e270d490799809d0., pid=9, masterSystemTime=1732607627877 2024-11-26T07:53:47,898 DEBUG [RS_OPEN_REGION-regionserver/719ac8e87449:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegionServer(2266): Finished post open deploy task for testBlockEvictionOnRegionMove1732607625523,,1732607625541.419e482dbb0367e2e270d490799809d0. 2024-11-26T07:53:47,898 INFO [RS_OPEN_REGION-regionserver/719ac8e87449:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] handler.AssignRegionHandler(153): Opened testBlockEvictionOnRegionMove1732607625523,,1732607625541.419e482dbb0367e2e270d490799809d0. 2024-11-26T07:53:47,898 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=419e482dbb0367e2e270d490799809d0, regionState=OPEN, openSeqNum=10, regionLocation=719ac8e87449,37023,1732607622257 2024-11-26T07:53:47,901 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-4-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=9, ppid=7, state=RUNNABLE, hasLock=false; OpenRegionProcedure 419e482dbb0367e2e270d490799809d0, server=719ac8e87449,37023,1732607622257 because future has completed 2024-11-26T07:53:47,905 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=9, resume processing ppid=7 2024-11-26T07:53:47,905 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=9, ppid=7, state=SUCCESS, hasLock=false; OpenRegionProcedure 419e482dbb0367e2e270d490799809d0, server=719ac8e87449,37023,1732607622257 in 178 msec 2024-11-26T07:53:47,907 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=7, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1732607625523, region=419e482dbb0367e2e270d490799809d0, REOPEN/MOVE in 533 msec 2024-11-26T07:53:47,931 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster 2024-11-26T07:53:47,931 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-26T07:53:47,932 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79) at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611) at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.io.hfile.TestBlockEvictionOnRegionMovement.tearDown(TestBlockEvictionOnRegionMovement.java:174) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-26T07:53:47,937 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T07:53:47,938 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T07:53:47,938 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-26T07:53:47,938 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2024-11-26T07:53:47,938 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=587878705, stopped=false 2024-11-26T07:53:47,939 INFO [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=719ac8e87449,36589,1732607621309 2024-11-26T07:53:47,983 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36589-0x101776bca1c0000, quorum=127.0.0.1:53139, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-26T07:53:47,983 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40465-0x101776bca1c0002, quorum=127.0.0.1:53139, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-26T07:53:47,983 DEBUG [pool-14-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37023-0x101776bca1c0001, quorum=127.0.0.1:53139, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-26T07:53:47,983 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40465-0x101776bca1c0002, quorum=127.0.0.1:53139, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-26T07:53:47,983 DEBUG [pool-14-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37023-0x101776bca1c0001, quorum=127.0.0.1:53139, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-26T07:53:47,983 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36589-0x101776bca1c0000, quorum=127.0.0.1:53139, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-26T07:53:47,983 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-26T07:53:47,984 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-26T07:53:47,984 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277) at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265) at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416) at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.io.hfile.TestBlockEvictionOnRegionMovement.tearDown(TestBlockEvictionOnRegionMovement.java:174) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-26T07:53:47,984 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T07:53:47,984 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:37023-0x101776bca1c0001, quorum=127.0.0.1:53139, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-26T07:53:47,984 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:40465-0x101776bca1c0002, quorum=127.0.0.1:53139, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-26T07:53:47,985 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '719ac8e87449,37023,1732607622257' ***** 2024-11-26T07:53:47,985 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:36589-0x101776bca1c0000, quorum=127.0.0.1:53139, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-26T07:53:47,985 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-11-26T07:53:47,985 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '719ac8e87449,40465,1732607622517' ***** 2024-11-26T07:53:47,985 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-11-26T07:53:47,985 INFO [RS:0;719ac8e87449:37023 {}] bucket.BucketCache(1738): Disabling cache 2024-11-26T07:53:47,985 INFO [RS:0;719ac8e87449:37023 {}] bucket.BucketCache(1762): Shutdown bucket cache: IO persistent=true; path to write=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/480cd986-aef2-fe93-7892-18b5e497e91c/bucket.persistence 2024-11-26T07:53:47,985 INFO [Time-limited test-BucketCacheWriter-1 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-1 exiting, cacheEnabled=false 2024-11-26T07:53:47,986 INFO [Time-limited test-BucketCacheWriter-2 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-2 exiting, cacheEnabled=false 2024-11-26T07:53:47,986 INFO [Time-limited test-BucketCacheWriter-0 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-0 exiting, cacheEnabled=false 2024-11-26T07:53:47,986 INFO [RS:0;719ac8e87449:37023 {}] bucket.BucketCache(1768): Shutting down cache persister thread. 2024-11-26T07:53:47,986 INFO [RS:1;719ac8e87449:40465 {}] bucket.BucketCache(1738): Disabling cache 2024-11-26T07:53:47,986 INFO [RS:1;719ac8e87449:40465 {}] bucket.BucketCache(1762): Shutdown bucket cache: IO persistent=true; path to write=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/480cd986-aef2-fe93-7892-18b5e497e91c/bucket.persistence 2024-11-26T07:53:47,986 INFO [Time-limited test-BucketCacheWriter-0 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-0 exiting, cacheEnabled=false 2024-11-26T07:53:47,986 WARN [bucket-cache-persister {}] bucket.BucketCachePersister(64): Interrupting BucketCachePersister thread. java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCachePersister.run(BucketCachePersister.java:45) ~[classes/:?] 2024-11-26T07:53:47,986 INFO [Time-limited test-BucketCacheWriter-2 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-2 exiting, cacheEnabled=false 2024-11-26T07:53:47,986 INFO [Time-limited test-BucketCacheWriter-1 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-1 exiting, cacheEnabled=false 2024-11-26T07:53:47,986 INFO [RS:1;719ac8e87449:40465 {}] bucket.BucketCache(1768): Shutting down cache persister thread. 2024-11-26T07:53:47,987 WARN [bucket-cache-persister {}] bucket.BucketCachePersister(64): Interrupting BucketCachePersister thread. java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCachePersister.run(BucketCachePersister.java:45) ~[classes/:?] 2024-11-26T07:53:47,996 DEBUG [RS:0;719ac8e87449:37023 {}] bucket.BucketCache(1487): Thread RS:0;719ac8e87449:37023 started persisting bucket cache to file 2024-11-26T07:53:47,996 DEBUG [RS:0;719ac8e87449:37023 {}] bucket.BucketCache(1494): Persist in new chunked persistence format. 2024-11-26T07:53:47,996 DEBUG [RS:0;719ac8e87449:37023 {}] bucket.BucketCache(1673): persistToFile: before persisting backing map size: 0, fullycachedFiles size: 0, chunkSize: 10000 2024-11-26T07:53:47,997 DEBUG [RS:1;719ac8e87449:40465 {}] bucket.BucketCache(1487): Thread RS:1;719ac8e87449:40465 started persisting bucket cache to file 2024-11-26T07:53:47,997 DEBUG [RS:1;719ac8e87449:40465 {}] bucket.BucketCache(1494): Persist in new chunked persistence format. 2024-11-26T07:53:47,997 DEBUG [RS:1;719ac8e87449:40465 {}] bucket.BucketCache(1673): persistToFile: before persisting backing map size: 0, fullycachedFiles size: 0, chunkSize: 10000 2024-11-26T07:53:48,000 DEBUG [RS:0;719ac8e87449:37023 {}] bucket.BucketCache(1680): persistToFile: after persisting backing map size: 0, fullycachedFiles size: 0 2024-11-26T07:53:48,000 DEBUG [RS:0;719ac8e87449:37023 {}] bucket.BucketCache(1498): PersistToFile: after persisting backing map size: 0, fullycachedFiles size: 0, file name: bucket.persistence1732607627996 2024-11-26T07:53:48,000 DEBUG [RS:0;719ac8e87449:37023 {}] bucket.BucketCache(1509): Thread RS:0;719ac8e87449:37023 finished persisting bucket cache to file, renaming 2024-11-26T07:53:48,000 DEBUG [RS:1;719ac8e87449:40465 {}] bucket.BucketCache(1680): persistToFile: after persisting backing map size: 0, fullycachedFiles size: 0 2024-11-26T07:53:48,001 DEBUG [RS:1;719ac8e87449:40465 {}] bucket.BucketCache(1498): PersistToFile: after persisting backing map size: 0, fullycachedFiles size: 0, file name: bucket.persistence1732607627997 2024-11-26T07:53:48,001 DEBUG [RS:1;719ac8e87449:40465 {}] bucket.BucketCache(1509): Thread RS:1;719ac8e87449:40465 finished persisting bucket cache to file, renaming 2024-11-26T07:53:48,001 INFO [RS:0;719ac8e87449:37023 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-26T07:53:48,001 INFO [RS:1;719ac8e87449:40465 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-26T07:53:48,001 INFO [RS:0;719ac8e87449:37023 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-26T07:53:48,001 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-26T07:53:48,001 INFO [RS:1;719ac8e87449:40465 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-26T07:53:48,001 INFO [RS:0;719ac8e87449:37023 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-26T07:53:48,001 INFO [RS:1;719ac8e87449:40465 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-26T07:53:48,001 INFO [RS:1;719ac8e87449:40465 {}] regionserver.HRegionServer(959): stopping server 719ac8e87449,40465,1732607622517 2024-11-26T07:53:48,001 INFO [RS:0;719ac8e87449:37023 {}] regionserver.HRegionServer(3091): Received CLOSE for 419e482dbb0367e2e270d490799809d0 2024-11-26T07:53:48,001 INFO [RS:1;719ac8e87449:40465 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-26T07:53:48,002 INFO [RS:1;719ac8e87449:40465 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:1;719ac8e87449:40465. 2024-11-26T07:53:48,002 DEBUG [RS:1;719ac8e87449:40465 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-26T07:53:48,002 DEBUG [RS:1;719ac8e87449:40465 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T07:53:48,002 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-26T07:53:48,002 INFO [RS:1;719ac8e87449:40465 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-26T07:53:48,002 INFO [RS:1;719ac8e87449:40465 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-26T07:53:48,002 INFO [RS:0;719ac8e87449:37023 {}] regionserver.HRegionServer(959): stopping server 719ac8e87449,37023,1732607622257 2024-11-26T07:53:48,002 INFO [RS:1;719ac8e87449:40465 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-26T07:53:48,002 INFO [RS:0;719ac8e87449:37023 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-26T07:53:48,002 INFO [RS:0;719ac8e87449:37023 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;719ac8e87449:37023. 2024-11-26T07:53:48,002 INFO [RS:1;719ac8e87449:40465 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-11-26T07:53:48,002 DEBUG [RS:0;719ac8e87449:37023 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-26T07:53:48,002 DEBUG [RS:0;719ac8e87449:37023 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T07:53:48,003 INFO [RS:0;719ac8e87449:37023 {}] regionserver.HRegionServer(1321): Waiting on 1 regions to close 2024-11-26T07:53:48,003 DEBUG [RS:0;719ac8e87449:37023 {}] regionserver.HRegionServer(1325): Online Regions={419e482dbb0367e2e270d490799809d0=testBlockEvictionOnRegionMove1732607625523,,1732607625541.419e482dbb0367e2e270d490799809d0.} 2024-11-26T07:53:48,003 INFO [RS_CLOSE_REGION-regionserver/719ac8e87449:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1662): Closing region testBlockEvictionOnRegionMove1732607625523,,1732607625541.419e482dbb0367e2e270d490799809d0. during a graceful stop, and cache persistence is on, so setting evict on close to false. 2024-11-26T07:53:48,003 DEBUG [RS:0;719ac8e87449:37023 {}] regionserver.HRegionServer(1351): Waiting on 419e482dbb0367e2e270d490799809d0 2024-11-26T07:53:48,003 DEBUG [RS_CLOSE_REGION-regionserver/719ac8e87449:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 419e482dbb0367e2e270d490799809d0, disabling compactions & flushes 2024-11-26T07:53:48,003 INFO [RS_CLOSE_REGION-regionserver/719ac8e87449:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testBlockEvictionOnRegionMove1732607625523,,1732607625541.419e482dbb0367e2e270d490799809d0. 2024-11-26T07:53:48,003 DEBUG [RS_CLOSE_REGION-regionserver/719ac8e87449:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testBlockEvictionOnRegionMove1732607625523,,1732607625541.419e482dbb0367e2e270d490799809d0. 2024-11-26T07:53:48,004 DEBUG [RS_CLOSE_REGION-regionserver/719ac8e87449:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testBlockEvictionOnRegionMove1732607625523,,1732607625541.419e482dbb0367e2e270d490799809d0. after waiting 1 ms 2024-11-26T07:53:48,004 DEBUG [RS_CLOSE_REGION-regionserver/719ac8e87449:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testBlockEvictionOnRegionMove1732607625523,,1732607625541.419e482dbb0367e2e270d490799809d0. 2024-11-26T07:53:48,004 INFO [RS:1;719ac8e87449:40465 {}] regionserver.HRegionServer(1321): Waiting on 1 regions to close 2024-11-26T07:53:48,004 DEBUG [RS:1;719ac8e87449:40465 {}] regionserver.HRegionServer(1325): Online Regions={1588230740=hbase:meta,,1.1588230740} 2024-11-26T07:53:48,004 DEBUG [RS:1;719ac8e87449:40465 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-26T07:53:48,004 INFO [RS_CLOSE_META-regionserver/719ac8e87449:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1662): Closing region hbase:meta,,1.1588230740 during a graceful stop, and cache persistence is on, so setting evict on close to false. 2024-11-26T07:53:48,004 DEBUG [RS_CLOSE_META-regionserver/719ac8e87449:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-26T07:53:48,005 INFO [RS_CLOSE_META-regionserver/719ac8e87449:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-26T07:53:48,005 DEBUG [RS_CLOSE_META-regionserver/719ac8e87449:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-26T07:53:48,005 DEBUG [RS_CLOSE_META-regionserver/719ac8e87449:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-26T07:53:48,005 DEBUG [RS_CLOSE_META-regionserver/719ac8e87449:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-26T07:53:48,005 DEBUG [StoreFileCloser-419e482dbb0367e2e270d490799809d0-family-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/480cd986-aef2-fe93-7892-18b5e497e91c/293d53df-6485-edb9-9b39-cf743ea8b2f2/data/default/testBlockEvictionOnRegionMove1732607625523/419e482dbb0367e2e270d490799809d0/family/e67d40e19a6d427cadef5c414992c907 2024-11-26T07:53:48,005 INFO [RS_CLOSE_META-regionserver/719ac8e87449:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=3.70 KB heapSize=6.72 KB 2024-11-26T07:53:48,011 DEBUG [RS_CLOSE_REGION-regionserver/719ac8e87449:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/480cd986-aef2-fe93-7892-18b5e497e91c/293d53df-6485-edb9-9b39-cf743ea8b2f2/data/default/testBlockEvictionOnRegionMove1732607625523/419e482dbb0367e2e270d490799809d0/recovered.edits/12.seqid, newMaxSeqId=12, maxSeqId=9 2024-11-26T07:53:48,012 INFO [RS_CLOSE_REGION-regionserver/719ac8e87449:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testBlockEvictionOnRegionMove1732607625523,,1732607625541.419e482dbb0367e2e270d490799809d0. 2024-11-26T07:53:48,012 DEBUG [RS_CLOSE_REGION-regionserver/719ac8e87449:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 419e482dbb0367e2e270d490799809d0: Waiting for close lock at 1732607628003Running coprocessor pre-close hooks at 1732607628003Disabling compacts and flushes for region at 1732607628003Disabling writes for close at 1732607628004 (+1 ms)Writing region close event to WAL at 1732607628006 (+2 ms)Running coprocessor post-close hooks at 1732607628012 (+6 ms)Closed at 1732607628012 2024-11-26T07:53:48,012 DEBUG [RS_CLOSE_REGION-regionserver/719ac8e87449:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testBlockEvictionOnRegionMove1732607625523,,1732607625541.419e482dbb0367e2e270d490799809d0. 2024-11-26T07:53:48,016 INFO [regionserver/719ac8e87449:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-26T07:53:48,017 INFO [regionserver/719ac8e87449:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-26T07:53:48,021 DEBUG [RS_CLOSE_META-regionserver/719ac8e87449:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFile$WriterFactory(308): Unable to set drop behind on c2c0ca15479d48e9b288000ffefcb88f 2024-11-26T07:53:48,045 DEBUG [RS_CLOSE_META-regionserver/719ac8e87449:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/480cd986-aef2-fe93-7892-18b5e497e91c/293d53df-6485-edb9-9b39-cf743ea8b2f2/data/hbase/meta/1588230740/.tmp/info/c2c0ca15479d48e9b288000ffefcb88f is 205, key is testBlockEvictionOnRegionMove1732607625523,,1732607625541.419e482dbb0367e2e270d490799809d0./info:regioninfo/1732607627898/Put/seqid=0 2024-11-26T07:53:48,046 INFO [RS_CLOSE_META-regionserver/719ac8e87449:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=3.48 KB at sequenceid=15 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/480cd986-aef2-fe93-7892-18b5e497e91c/293d53df-6485-edb9-9b39-cf743ea8b2f2/data/hbase/meta/1588230740/.tmp/info/c2c0ca15479d48e9b288000ffefcb88f 2024-11-26T07:53:48,059 DEBUG [RS_CLOSE_META-regionserver/719ac8e87449:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 36a33ce3643c4322bb6ecac19bd1ce4f 2024-11-26T07:53:48,078 DEBUG [RS_CLOSE_META-regionserver/719ac8e87449:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/480cd986-aef2-fe93-7892-18b5e497e91c/293d53df-6485-edb9-9b39-cf743ea8b2f2/data/hbase/meta/1588230740/.tmp/ns/36a33ce3643c4322bb6ecac19bd1ce4f is 43, key is default/ns:d/1732607625246/Put/seqid=0 2024-11-26T07:53:48,079 INFO [RS_CLOSE_META-regionserver/719ac8e87449:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=74 B at sequenceid=15 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/480cd986-aef2-fe93-7892-18b5e497e91c/293d53df-6485-edb9-9b39-cf743ea8b2f2/data/hbase/meta/1588230740/.tmp/ns/36a33ce3643c4322bb6ecac19bd1ce4f 2024-11-26T07:53:48,089 DEBUG [RS_CLOSE_META-regionserver/719ac8e87449:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 6240d4e13e624bf59c3bc793847247e6 2024-11-26T07:53:48,102 DEBUG [RS_CLOSE_META-regionserver/719ac8e87449:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/480cd986-aef2-fe93-7892-18b5e497e91c/293d53df-6485-edb9-9b39-cf743ea8b2f2/data/hbase/meta/1588230740/.tmp/table/6240d4e13e624bf59c3bc793847247e6 is 78, key is testBlockEvictionOnRegionMove1732607625523/table:state/1732607625984/Put/seqid=0 2024-11-26T07:53:48,103 INFO [RS_CLOSE_META-regionserver/719ac8e87449:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=148 B at sequenceid=15 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/480cd986-aef2-fe93-7892-18b5e497e91c/293d53df-6485-edb9-9b39-cf743ea8b2f2/data/hbase/meta/1588230740/.tmp/table/6240d4e13e624bf59c3bc793847247e6 2024-11-26T07:53:48,106 DEBUG [RS_CLOSE_META-regionserver/719ac8e87449:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/480cd986-aef2-fe93-7892-18b5e497e91c/293d53df-6485-edb9-9b39-cf743ea8b2f2/data/hbase/meta/1588230740/.tmp/info/c2c0ca15479d48e9b288000ffefcb88f as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/480cd986-aef2-fe93-7892-18b5e497e91c/293d53df-6485-edb9-9b39-cf743ea8b2f2/data/hbase/meta/1588230740/info/c2c0ca15479d48e9b288000ffefcb88f 2024-11-26T07:53:48,108 DEBUG [RS_CLOSE_META-regionserver/719ac8e87449:0-0 {event_type=M_RS_CLOSE_META}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/480cd986-aef2-fe93-7892-18b5e497e91c/293d53df-6485-edb9-9b39-cf743ea8b2f2/data/hbase/meta/1588230740/info/c2c0ca15479d48e9b288000ffefcb88f, delay=909 ms 2024-11-26T07:53:48,110 INFO [RS_CLOSE_META-regionserver/719ac8e87449:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/480cd986-aef2-fe93-7892-18b5e497e91c/293d53df-6485-edb9-9b39-cf743ea8b2f2/data/hbase/meta/1588230740/info/c2c0ca15479d48e9b288000ffefcb88f, entries=15, sequenceid=15, filesize=7.6 K 2024-11-26T07:53:48,110 DEBUG [RS_CLOSE_META-regionserver/719ac8e87449:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/480cd986-aef2-fe93-7892-18b5e497e91c/293d53df-6485-edb9-9b39-cf743ea8b2f2/data/hbase/meta/1588230740/.tmp/ns/36a33ce3643c4322bb6ecac19bd1ce4f as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/480cd986-aef2-fe93-7892-18b5e497e91c/293d53df-6485-edb9-9b39-cf743ea8b2f2/data/hbase/meta/1588230740/ns/36a33ce3643c4322bb6ecac19bd1ce4f 2024-11-26T07:53:48,111 DEBUG [RS_CLOSE_META-regionserver/719ac8e87449:0-0 {event_type=M_RS_CLOSE_META}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/480cd986-aef2-fe93-7892-18b5e497e91c/293d53df-6485-edb9-9b39-cf743ea8b2f2/data/hbase/meta/1588230740/ns/36a33ce3643c4322bb6ecac19bd1ce4f, delay=904 ms 2024-11-26T07:53:48,113 INFO [RS_CLOSE_META-regionserver/719ac8e87449:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/480cd986-aef2-fe93-7892-18b5e497e91c/293d53df-6485-edb9-9b39-cf743ea8b2f2/data/hbase/meta/1588230740/ns/36a33ce3643c4322bb6ecac19bd1ce4f, entries=2, sequenceid=15, filesize=5.0 K 2024-11-26T07:53:48,114 DEBUG [RS_CLOSE_META-regionserver/719ac8e87449:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/480cd986-aef2-fe93-7892-18b5e497e91c/293d53df-6485-edb9-9b39-cf743ea8b2f2/data/hbase/meta/1588230740/.tmp/table/6240d4e13e624bf59c3bc793847247e6 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/480cd986-aef2-fe93-7892-18b5e497e91c/293d53df-6485-edb9-9b39-cf743ea8b2f2/data/hbase/meta/1588230740/table/6240d4e13e624bf59c3bc793847247e6 2024-11-26T07:53:48,115 DEBUG [RS_CLOSE_META-regionserver/719ac8e87449:0-0 {event_type=M_RS_CLOSE_META}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/480cd986-aef2-fe93-7892-18b5e497e91c/293d53df-6485-edb9-9b39-cf743ea8b2f2/data/hbase/meta/1588230740/table/6240d4e13e624bf59c3bc793847247e6, delay=933 ms 2024-11-26T07:53:48,116 INFO [RS_CLOSE_META-regionserver/719ac8e87449:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/480cd986-aef2-fe93-7892-18b5e497e91c/293d53df-6485-edb9-9b39-cf743ea8b2f2/data/hbase/meta/1588230740/table/6240d4e13e624bf59c3bc793847247e6, entries=2, sequenceid=15, filesize=5.3 K 2024-11-26T07:53:48,117 INFO [RS_CLOSE_META-regionserver/719ac8e87449:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~3.70 KB/3789, heapSize ~6.42 KB/6576, currentSize=0 B/0 for 1588230740 in 112ms, sequenceid=15, compaction requested=false 2024-11-26T07:53:48,117 DEBUG [RS_CLOSE_META-regionserver/719ac8e87449:0-0 {event_type=M_RS_CLOSE_META}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta' 2024-11-26T07:53:48,119 DEBUG [StoreFileCloser-1588230740-info-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/480cd986-aef2-fe93-7892-18b5e497e91c/293d53df-6485-edb9-9b39-cf743ea8b2f2/data/hbase/meta/1588230740/info/c2c0ca15479d48e9b288000ffefcb88f 2024-11-26T07:53:48,120 DEBUG [StoreFileCloser-1588230740-ns-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/480cd986-aef2-fe93-7892-18b5e497e91c/293d53df-6485-edb9-9b39-cf743ea8b2f2/data/hbase/meta/1588230740/ns/36a33ce3643c4322bb6ecac19bd1ce4f 2024-11-26T07:53:48,121 DEBUG [StoreFileCloser-1588230740-table-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/480cd986-aef2-fe93-7892-18b5e497e91c/293d53df-6485-edb9-9b39-cf743ea8b2f2/data/hbase/meta/1588230740/table/6240d4e13e624bf59c3bc793847247e6 2024-11-26T07:53:48,127 DEBUG [RS_CLOSE_META-regionserver/719ac8e87449:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/480cd986-aef2-fe93-7892-18b5e497e91c/293d53df-6485-edb9-9b39-cf743ea8b2f2/data/hbase/meta/1588230740/recovered.edits/18.seqid, newMaxSeqId=18, maxSeqId=1 2024-11-26T07:53:48,127 DEBUG [RS_CLOSE_META-regionserver/719ac8e87449:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-26T07:53:48,128 INFO [RS_CLOSE_META-regionserver/719ac8e87449:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-26T07:53:48,128 DEBUG [RS_CLOSE_META-regionserver/719ac8e87449:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1732607628004Running coprocessor pre-close hooks at 1732607628004Disabling compacts and flushes for region at 1732607628004Disabling writes for close at 1732607628005 (+1 ms)Obtaining lock to block concurrent updates at 1732607628005Preparing flush snapshotting stores in 1588230740 at 1732607628005Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=3789, getHeapSize=6816, getOffHeapSize=0, getCellsCount=27 at 1732607628006 (+1 ms)Flushing stores of hbase:meta,,1.1588230740 at 1732607628006Flushing 1588230740/info: creating writer at 1732607628006Flushing 1588230740/info: appending metadata at 1732607628043 (+37 ms)Flushing 1588230740/info: closing flushed file at 1732607628043Flushing 1588230740/ns: creating writer at 1732607628050 (+7 ms)Flushing 1588230740/ns: appending metadata at 1732607628077 (+27 ms)Flushing 1588230740/ns: closing flushed file at 1732607628077Flushing 1588230740/table: creating writer at 1732607628083 (+6 ms)Flushing 1588230740/table: appending metadata at 1732607628101 (+18 ms)Flushing 1588230740/table: closing flushed file at 1732607628101Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5399e21: reopening flushed file at 1732607628105 (+4 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5e419e2f: reopening flushed file at 1732607628110 (+5 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@7a9e5f9a: reopening flushed file at 1732607628113 (+3 ms)Finished flush of dataSize ~3.70 KB/3789, heapSize ~6.42 KB/6576, currentSize=0 B/0 for 1588230740 in 112ms, sequenceid=15, compaction requested=false at 1732607628117 (+4 ms)Writing region close event to WAL at 1732607628122 (+5 ms)Running coprocessor post-close hooks at 1732607628127 (+5 ms)Closed at 1732607628128 (+1 ms) 2024-11-26T07:53:48,128 DEBUG [RS_CLOSE_META-regionserver/719ac8e87449:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-11-26T07:53:48,203 INFO [RS:0;719ac8e87449:37023 {}] regionserver.HRegionServer(976): stopping server 719ac8e87449,37023,1732607622257; all regions closed. 2024-11-26T07:53:48,204 INFO [RS:1;719ac8e87449:40465 {}] regionserver.HRegionServer(976): stopping server 719ac8e87449,40465,1732607622517; all regions closed. 2024-11-26T07:53:48,208 DEBUG [RS:0;719ac8e87449:37023 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/480cd986-aef2-fe93-7892-18b5e497e91c/293d53df-6485-edb9-9b39-cf743ea8b2f2/oldWALs 2024-11-26T07:53:48,208 DEBUG [RS:1;719ac8e87449:40465 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/480cd986-aef2-fe93-7892-18b5e497e91c/293d53df-6485-edb9-9b39-cf743ea8b2f2/oldWALs 2024-11-26T07:53:48,208 INFO [RS:0;719ac8e87449:37023 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 719ac8e87449%2C37023%2C1732607622257:(num 1732607624576) 2024-11-26T07:53:48,208 DEBUG [RS:0;719ac8e87449:37023 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T07:53:48,208 INFO [RS:1;719ac8e87449:40465 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 719ac8e87449%2C40465%2C1732607622517.meta:.meta(num 1732607625072) 2024-11-26T07:53:48,208 INFO [RS:0;719ac8e87449:37023 {}] regionserver.LeaseManager(133): Closed leases 2024-11-26T07:53:48,209 INFO [RS:0;719ac8e87449:37023 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-26T07:53:48,209 INFO [RS:0;719ac8e87449:37023 {}] hbase.ChoreService(370): Chore service for: regionserver/719ac8e87449:0 had [ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-11-26T07:53:48,209 INFO [RS:0;719ac8e87449:37023 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-26T07:53:48,209 INFO [RS:0;719ac8e87449:37023 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-26T07:53:48,209 INFO [RS:0;719ac8e87449:37023 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-26T07:53:48,209 INFO [RS:0;719ac8e87449:37023 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-26T07:53:48,210 INFO [RS:0;719ac8e87449:37023 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:37023 2024-11-26T07:53:48,210 DEBUG [RS:1;719ac8e87449:40465 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/480cd986-aef2-fe93-7892-18b5e497e91c/293d53df-6485-edb9-9b39-cf743ea8b2f2/oldWALs 2024-11-26T07:53:48,210 INFO [RS:1;719ac8e87449:40465 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 719ac8e87449%2C40465%2C1732607622517:(num 1732607624576) 2024-11-26T07:53:48,210 DEBUG [RS:1;719ac8e87449:40465 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T07:53:48,210 INFO [RS:1;719ac8e87449:40465 {}] regionserver.LeaseManager(133): Closed leases 2024-11-26T07:53:48,210 INFO [RS:1;719ac8e87449:40465 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-26T07:53:48,211 INFO [RS:1;719ac8e87449:40465 {}] hbase.ChoreService(370): Chore service for: regionserver/719ac8e87449:0 had [ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS] on shutdown 2024-11-26T07:53:48,211 INFO [RS:1;719ac8e87449:40465 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-26T07:53:48,211 INFO [regionserver/719ac8e87449:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-26T07:53:48,211 INFO [RS:1;719ac8e87449:40465 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:40465 2024-11-26T07:53:48,212 INFO [regionserver/719ac8e87449:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-26T07:53:48,233 DEBUG [pool-14-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37023-0x101776bca1c0001, quorum=127.0.0.1:53139, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/719ac8e87449,37023,1732607622257 2024-11-26T07:53:48,233 INFO [RS:0;719ac8e87449:37023 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-26T07:53:48,233 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36589-0x101776bca1c0000, quorum=127.0.0.1:53139, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-26T07:53:48,241 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40465-0x101776bca1c0002, quorum=127.0.0.1:53139, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/719ac8e87449,40465,1732607622517 2024-11-26T07:53:48,241 INFO [RS:1;719ac8e87449:40465 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-26T07:53:48,242 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [719ac8e87449,40465,1732607622517] 2024-11-26T07:53:48,258 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/719ac8e87449,40465,1732607622517 already deleted, retry=false 2024-11-26T07:53:48,258 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; 719ac8e87449,40465,1732607622517 expired; onlineServers=1 2024-11-26T07:53:48,258 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [719ac8e87449,37023,1732607622257] 2024-11-26T07:53:48,266 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/719ac8e87449,37023,1732607622257 already deleted, retry=false 2024-11-26T07:53:48,266 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; 719ac8e87449,37023,1732607622257 expired; onlineServers=0 2024-11-26T07:53:48,267 INFO [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master '719ac8e87449,36589,1732607621309' ***** 2024-11-26T07:53:48,267 INFO [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0 2024-11-26T07:53:48,267 INFO [M:0;719ac8e87449:36589 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-26T07:53:48,267 INFO [M:0;719ac8e87449:36589 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-26T07:53:48,267 DEBUG [M:0;719ac8e87449:36589 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2024-11-26T07:53:48,267 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2024-11-26T07:53:48,267 DEBUG [M:0;719ac8e87449:36589 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2024-11-26T07:53:48,267 DEBUG [master/719ac8e87449:0:becomeActiveMaster-HFileCleaner.small.0-1732607624189 {}] cleaner.HFileCleaner(306): Exit Thread[master/719ac8e87449:0:becomeActiveMaster-HFileCleaner.small.0-1732607624189,5,FailOnTimeoutGroup] 2024-11-26T07:53:48,267 DEBUG [master/719ac8e87449:0:becomeActiveMaster-HFileCleaner.large.0-1732607624186 {}] cleaner.HFileCleaner(306): Exit Thread[master/719ac8e87449:0:becomeActiveMaster-HFileCleaner.large.0-1732607624186,5,FailOnTimeoutGroup] 2024-11-26T07:53:48,267 INFO [M:0;719ac8e87449:36589 {}] hbase.ChoreService(370): Chore service for: master/719ac8e87449:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown 2024-11-26T07:53:48,268 INFO [M:0;719ac8e87449:36589 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-26T07:53:48,268 DEBUG [M:0;719ac8e87449:36589 {}] master.HMaster(1795): Stopping service threads 2024-11-26T07:53:48,268 INFO [M:0;719ac8e87449:36589 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2024-11-26T07:53:48,268 INFO [M:0;719ac8e87449:36589 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-26T07:53:48,269 INFO [M:0;719ac8e87449:36589 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-26T07:53:48,269 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2024-11-26T07:53:48,275 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36589-0x101776bca1c0000, quorum=127.0.0.1:53139, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-11-26T07:53:48,275 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36589-0x101776bca1c0000, quorum=127.0.0.1:53139, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-26T07:53:48,275 DEBUG [M:0;719ac8e87449:36589 {}] zookeeper.ZKUtil(347): master:36589-0x101776bca1c0000, quorum=127.0.0.1:53139, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error) 2024-11-26T07:53:48,275 WARN [M:0;719ac8e87449:36589 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null 2024-11-26T07:53:48,275 INFO [M:0;719ac8e87449:36589 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/480cd986-aef2-fe93-7892-18b5e497e91c/293d53df-6485-edb9-9b39-cf743ea8b2f2/.lastflushedseqids 2024-11-26T07:53:48,285 INFO [M:0;719ac8e87449:36589 {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-26T07:53:48,285 INFO [M:0;719ac8e87449:36589 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2024-11-26T07:53:48,286 DEBUG [M:0;719ac8e87449:36589 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-26T07:53:48,286 INFO [M:0;719ac8e87449:36589 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-26T07:53:48,286 DEBUG [M:0;719ac8e87449:36589 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-26T07:53:48,286 DEBUG [M:0;719ac8e87449:36589 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-26T07:53:48,286 DEBUG [M:0;719ac8e87449:36589 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-26T07:53:48,286 INFO [M:0;719ac8e87449:36589 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=34.69 KB heapSize=43.02 KB 2024-11-26T07:53:48,298 DEBUG [M:0;719ac8e87449:36589 {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 6f28e7b155a746d99f9134b2f6ddd79d 2024-11-26T07:53:48,310 DEBUG [M:0;719ac8e87449:36589 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/480cd986-aef2-fe93-7892-18b5e497e91c/293d53df-6485-edb9-9b39-cf743ea8b2f2/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/6f28e7b155a746d99f9134b2f6ddd79d is 82, key is hbase:meta,,1/info:regioninfo/1732607625144/Put/seqid=0 2024-11-26T07:53:48,311 INFO [M:0;719ac8e87449:36589 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=504 B at sequenceid=84 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/480cd986-aef2-fe93-7892-18b5e497e91c/293d53df-6485-edb9-9b39-cf743ea8b2f2/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/6f28e7b155a746d99f9134b2f6ddd79d 2024-11-26T07:53:48,320 DEBUG [M:0;719ac8e87449:36589 {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 3d5614d10f42428191642f5a75723eab 2024-11-26T07:53:48,339 DEBUG [M:0;719ac8e87449:36589 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/480cd986-aef2-fe93-7892-18b5e497e91c/293d53df-6485-edb9-9b39-cf743ea8b2f2/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/3d5614d10f42428191642f5a75723eab is 779, key is \x00\x00\x00\x00\x00\x00\x00\x04/proc:d/1732607625990/Put/seqid=0 2024-11-26T07:53:48,340 INFO [M:0;719ac8e87449:36589 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=34.07 KB at sequenceid=84 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/480cd986-aef2-fe93-7892-18b5e497e91c/293d53df-6485-edb9-9b39-cf743ea8b2f2/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/3d5614d10f42428191642f5a75723eab 2024-11-26T07:53:48,349 DEBUG [M:0;719ac8e87449:36589 {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on b12d570c18ff4dc1926809553e1bb58c 2024-11-26T07:53:48,350 DEBUG [pool-14-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37023-0x101776bca1c0001, quorum=127.0.0.1:53139, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-26T07:53:48,350 DEBUG [pool-14-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37023-0x101776bca1c0001, quorum=127.0.0.1:53139, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-26T07:53:48,351 INFO [RS:0;719ac8e87449:37023 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-26T07:53:48,351 INFO [RS:0;719ac8e87449:37023 {}] regionserver.HRegionServer(1031): Exiting; stopping=719ac8e87449,37023,1732607622257; zookeeper connection closed. 2024-11-26T07:53:48,351 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@66070e21 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@66070e21 2024-11-26T07:53:48,358 INFO [RS:1;719ac8e87449:40465 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-26T07:53:48,358 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40465-0x101776bca1c0002, quorum=127.0.0.1:53139, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-26T07:53:48,358 INFO [RS:1;719ac8e87449:40465 {}] regionserver.HRegionServer(1031): Exiting; stopping=719ac8e87449,40465,1732607622517; zookeeper connection closed. 2024-11-26T07:53:48,358 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40465-0x101776bca1c0002, quorum=127.0.0.1:53139, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-26T07:53:48,358 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@49f33acc {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@49f33acc 2024-11-26T07:53:48,359 INFO [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 2 regionserver(s) complete 2024-11-26T07:53:48,366 DEBUG [M:0;719ac8e87449:36589 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/480cd986-aef2-fe93-7892-18b5e497e91c/293d53df-6485-edb9-9b39-cf743ea8b2f2/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/b12d570c18ff4dc1926809553e1bb58c is 69, key is 719ac8e87449,37023,1732607622257/rs:state/1732607624241/Put/seqid=0 2024-11-26T07:53:48,367 INFO [M:0;719ac8e87449:36589 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=130 B at sequenceid=84 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/480cd986-aef2-fe93-7892-18b5e497e91c/293d53df-6485-edb9-9b39-cf743ea8b2f2/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/b12d570c18ff4dc1926809553e1bb58c 2024-11-26T07:53:48,369 DEBUG [M:0;719ac8e87449:36589 {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/480cd986-aef2-fe93-7892-18b5e497e91c/293d53df-6485-edb9-9b39-cf743ea8b2f2/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/6f28e7b155a746d99f9134b2f6ddd79d as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/480cd986-aef2-fe93-7892-18b5e497e91c/293d53df-6485-edb9-9b39-cf743ea8b2f2/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/6f28e7b155a746d99f9134b2f6ddd79d 2024-11-26T07:53:48,371 INFO [M:0;719ac8e87449:36589 {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/480cd986-aef2-fe93-7892-18b5e497e91c/293d53df-6485-edb9-9b39-cf743ea8b2f2/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/6f28e7b155a746d99f9134b2f6ddd79d, entries=8, sequenceid=84, filesize=5.5 K 2024-11-26T07:53:48,372 DEBUG [M:0;719ac8e87449:36589 {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/480cd986-aef2-fe93-7892-18b5e497e91c/293d53df-6485-edb9-9b39-cf743ea8b2f2/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/3d5614d10f42428191642f5a75723eab as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/480cd986-aef2-fe93-7892-18b5e497e91c/293d53df-6485-edb9-9b39-cf743ea8b2f2/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/3d5614d10f42428191642f5a75723eab 2024-11-26T07:53:48,374 INFO [M:0;719ac8e87449:36589 {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/480cd986-aef2-fe93-7892-18b5e497e91c/293d53df-6485-edb9-9b39-cf743ea8b2f2/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/3d5614d10f42428191642f5a75723eab, entries=9, sequenceid=84, filesize=6.5 K 2024-11-26T07:53:48,375 DEBUG [M:0;719ac8e87449:36589 {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/480cd986-aef2-fe93-7892-18b5e497e91c/293d53df-6485-edb9-9b39-cf743ea8b2f2/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/b12d570c18ff4dc1926809553e1bb58c as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/480cd986-aef2-fe93-7892-18b5e497e91c/293d53df-6485-edb9-9b39-cf743ea8b2f2/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/b12d570c18ff4dc1926809553e1bb58c 2024-11-26T07:53:48,377 INFO [M:0;719ac8e87449:36589 {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/480cd986-aef2-fe93-7892-18b5e497e91c/293d53df-6485-edb9-9b39-cf743ea8b2f2/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/b12d570c18ff4dc1926809553e1bb58c, entries=2, sequenceid=84, filesize=5.1 K 2024-11-26T07:53:48,378 INFO [M:0;719ac8e87449:36589 {}] regionserver.HRegion(3140): Finished flush of dataSize ~34.69 KB/35518, heapSize ~42.73 KB/43752, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 92ms, sequenceid=84, compaction requested=false 2024-11-26T07:53:48,382 INFO [M:0;719ac8e87449:36589 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-26T07:53:48,382 DEBUG [M:0;719ac8e87449:36589 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1732607628286Disabling compacts and flushes for region at 1732607628286Disabling writes for close at 1732607628286Obtaining lock to block concurrent updates at 1732607628286Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1732607628286Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=35518, getHeapSize=43992, getOffHeapSize=0, getCellsCount=99 at 1732607628287 (+1 ms)Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1732607628287Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1732607628287Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1732607628310 (+23 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1732607628310Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1732607628313 (+3 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1732607628338 (+25 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1732607628338Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1732607628342 (+4 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1732607628365 (+23 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1732607628365Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@60d10f2f: reopening flushed file at 1732607628369 (+4 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@2d39f40: reopening flushed file at 1732607628372 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@2be31822: reopening flushed file at 1732607628374 (+2 ms)Finished flush of dataSize ~34.69 KB/35518, heapSize ~42.73 KB/43752, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 92ms, sequenceid=84, compaction requested=false at 1732607628378 (+4 ms)Writing region close event to WAL at 1732607628382 (+4 ms)Closed at 1732607628382 2024-11-26T07:53:48,384 INFO [M:0;719ac8e87449:36589 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down. 2024-11-26T07:53:48,385 INFO [M:0;719ac8e87449:36589 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:36589 2024-11-26T07:53:48,385 INFO [M:0;719ac8e87449:36589 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-26T07:53:48,385 INFO [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-26T07:53:48,500 INFO [M:0;719ac8e87449:36589 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-26T07:53:48,500 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36589-0x101776bca1c0000, quorum=127.0.0.1:53139, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-26T07:53:48,500 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36589-0x101776bca1c0000, quorum=127.0.0.1:53139, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-26T07:53:48,505 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers 2024-11-26T07:53:48,527 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down 2024-11-26T07:53:48,528 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-26T07:53:48,536 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: io.hfile.TestBlockEvictionOnRegionMovement#testBlockEvictionOnRegionMove Thread=49 (was 12) Potentially hanging thread: ForkJoinPool.commonPool-worker-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.ForkJoinPool.awaitWork(ForkJoinPool.java:1724) java.base@17.0.11/java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1623) java.base@17.0.11/java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:165) Potentially hanging thread: HMaster-EventLoopGroup-1-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: org.apache.hadoop.fs.FileSystem$Statistics$StatisticsDataReferenceCleaner java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:155) java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:176) app//org.apache.hadoop.fs.FileSystem$Statistics$StatisticsDataReferenceCleaner.run(FileSystem.java:4171) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: region-location-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1170) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Monitor thread for TaskMonitor java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hbase.monitoring.TaskMonitor$MonitorRunnable.run(TaskMonitor.java:325) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-4-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: regionserver/719ac8e87449:0.procedureResultReporter java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.RemoteProcedureResultReporter.run(RemoteProcedureResultReporter.java:75) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: hfile-prefetch-1732607628112 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1177) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: hfile-prefetch-1732607626343 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: SessionTracker java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.zookeeper.server.SessionTrackerImpl.run(SessionTrackerImpl.java:163) Potentially hanging thread: HMaster-EventLoopGroup-1-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-4-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: hfile-prefetch-1732607627885 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1177) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RpcClient-timer-pool-0 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.waitForNextTick(HashedWheelTimer.java:598) app//org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:494) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: master/719ac8e87449:0:becomeActiveMaster-MemStoreChunkPool Statistics java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: SnapshotHandlerChoreCleaner java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: regionserver/719ac8e87449:0.procedureResultReporter java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.RemoteProcedureResultReporter.run(RemoteProcedureResultReporter.java:75) Potentially hanging thread: HMaster-EventLoopGroup-1-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HBase-Metrics2-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Time-limited test.named-queue-events-pool-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) app//com.lmax.disruptor.BlockingWaitStrategy.waitFor(BlockingWaitStrategy.java:47) app//com.lmax.disruptor.ProcessingSequenceBarrier.waitFor(ProcessingSequenceBarrier.java:56) app//com.lmax.disruptor.BatchEventProcessor.processEvents(BatchEventProcessor.java:159) app//com.lmax.disruptor.BatchEventProcessor.run(BatchEventProcessor.java:125) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-4 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ForkJoinPool.commonPool-worker-3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkUntil(LockSupport.java:410) java.base@17.0.11/java.util.concurrent.ForkJoinPool.awaitWork(ForkJoinPool.java:1726) java.base@17.0.11/java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1623) java.base@17.0.11/java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:165) Potentially hanging thread: weak-ref-cleaner-strictcontextstorage java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:155) java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:176) app//io.opentelemetry.context.StrictContextStorage$PendingScopes.run(StrictContextStorage.java:269) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-4-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: region-location-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1170) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ForkJoinPool.commonPool-worker-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.ForkJoinPool.awaitWork(ForkJoinPool.java:1724) java.base@17.0.11/java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1623) java.base@17.0.11/java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:165) Potentially hanging thread: Async-Client-Retry-Timer-pool-0 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.waitForNextTick(HashedWheelTimer.java:598) app//org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:494) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-3-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: hfile-prefetch-1732607628108 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1177) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-3-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: master/719ac8e87449:0:becomeActiveMaster-MemStoreChunkPool Statistics java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Idle-Rpc-Conn-Sweeper-pool-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Timer for 'HBase' metrics system java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.util.TimerThread.mainLoop(Timer.java:563) java.base@17.0.11/java.util.TimerThread.run(Timer.java:516) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=346 (was 287) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=428 (was 407) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=10969 (was 11334) 2024-11-26T07:53:48,541 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: io.hfile.TestBlockEvictionOnRegionMovement#testBlockEvictionOnGracefulStop Thread=49, OpenFileDescriptor=346, MaxFileDescriptor=1048576, SystemLoadAverage=428, ProcessCount=11, AvailableMemoryMB=10969 2024-11-26T07:53:48,542 INFO [Time-limited test {}] hbase.HBaseTestingUtil(401): System.getProperty("hadoop.log.dir") already set to: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/480cd986-aef2-fe93-7892-18b5e497e91c/hadoop-log-dir so I do NOT create it in target/test-data/3acb3e4a-dea5-ea8d-f2d0-bb92c503e71b 2024-11-26T07:53:48,542 INFO [Time-limited test {}] hbase.HBaseTestingUtil(401): System.getProperty("hadoop.tmp.dir") already set to: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/480cd986-aef2-fe93-7892-18b5e497e91c/hadoop-tmp-dir so I do NOT create it in target/test-data/3acb3e4a-dea5-ea8d-f2d0-bb92c503e71b 2024-11-26T07:53:48,542 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-26T07:53:48,546 INFO [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3acb3e4a-dea5-ea8d-f2d0-bb92c503e71b/cluster_92219bda-488b-f4a3-95d8-e24219f3bf00, deleteOnExit=true 2024-11-26T07:53:48,581 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3acb3e4a-dea5-ea8d-f2d0-bb92c503e71b/cluster_92219bda-488b-f4a3-95d8-e24219f3bf00/zookeeper_0, clientPort=64716, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3acb3e4a-dea5-ea8d-f2d0-bb92c503e71b/cluster_92219bda-488b-f4a3-95d8-e24219f3bf00/zookeeper_0/version-2, dataDirSize=457 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3acb3e4a-dea5-ea8d-f2d0-bb92c503e71b/cluster_92219bda-488b-f4a3-95d8-e24219f3bf00/zookeeper_0/version-2, dataLogSize=457 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0 2024-11-26T07:53:48,582 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=64716 2024-11-26T07:53:48,595 INFO [Time-limited test {}] util.FSUtils(489): Created version file at /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/480cd986-aef2-fe93-7892-18b5e497e91c/293d53df-6485-edb9-9b39-cf743ea8b2f2 with version=8 2024-11-26T07:53:48,595 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1141): The hbase.fs.tmp.dir is set to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/480cd986-aef2-fe93-7892-18b5e497e91c/293d53df-6485-edb9-9b39-cf743ea8b2f2/hbase-staging 2024-11-26T07:53:48,597 INFO [Time-limited test {}] client.ConnectionUtils(128): master/719ac8e87449:0 server-side Connection retries=45 2024-11-26T07:53:48,597 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-26T07:53:48,598 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-26T07:53:48,598 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-26T07:53:48,598 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-26T07:53:48,598 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-26T07:53:48,598 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService 2024-11-26T07:53:48,598 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-26T07:53:48,599 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:36471 2024-11-26T07:53:48,600 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:36471 connecting to ZooKeeper ensemble=127.0.0.1:64716 2024-11-26T07:53:48,640 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:364710x0, quorum=127.0.0.1:64716, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-26T07:53:48,641 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:36471-0x101776be7e90000 connected 2024-11-26T07:53:48,716 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-26T07:53:48,716 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-26T07:53:48,717 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:36471-0x101776be7e90000, quorum=127.0.0.1:64716, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-26T07:53:48,717 INFO [Time-limited test {}] master.HMaster(525): hbase.rootdir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/480cd986-aef2-fe93-7892-18b5e497e91c/293d53df-6485-edb9-9b39-cf743ea8b2f2, hbase.cluster.distributed=false 2024-11-26T07:53:48,720 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:36471-0x101776be7e90000, quorum=127.0.0.1:64716, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-26T07:53:48,721 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=36471 2024-11-26T07:53:48,721 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=36471 2024-11-26T07:53:48,721 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=36471 2024-11-26T07:53:48,721 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=36471 2024-11-26T07:53:48,722 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=36471 2024-11-26T07:53:48,743 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/719ac8e87449:0 server-side Connection retries=45 2024-11-26T07:53:48,744 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-26T07:53:48,744 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-26T07:53:48,744 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-26T07:53:48,744 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-26T07:53:48,744 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-26T07:53:48,744 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-26T07:53:48,744 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-26T07:53:48,745 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:43013 2024-11-26T07:53:48,747 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:43013 connecting to ZooKeeper ensemble=127.0.0.1:64716 2024-11-26T07:53:48,748 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-26T07:53:48,748 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-26T07:53:48,757 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:430130x0, quorum=127.0.0.1:64716, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-26T07:53:48,758 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:43013-0x101776be7e90001 connected 2024-11-26T07:53:48,758 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:43013-0x101776be7e90001, quorum=127.0.0.1:64716, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-26T07:53:48,759 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-26T07:53:48,762 INFO [Time-limited test {}] bucket.FileIOEngine(99): Allocating cache 400 MB, on the path:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3acb3e4a-dea5-ea8d-f2d0-bb92c503e71b/bucket.cache 2024-11-26T07:53:48,762 INFO [Time-limited test {}] bucket.BucketCache(364): Instantiating BucketCache with acceptableFactor: 0.95, minFactor: 0.85, extraFreeFactor: 0.1, singleFactor: 0.25, multiFactor: 0.5, memoryFactor: 0.25, useStrongRef: false 2024-11-26T07:53:48,762 INFO [Time-limited test {}] bucket.BucketCache(478): Starting BucketCachePersisterThread 2024-11-26T07:53:48,762 INFO [Time-limited test {}] bucket.BucketCachePersister(38): BucketCachePersister started with interval: 100 2024-11-26T07:53:48,766 INFO [Thread-118 {}] bucket.BucketCache(1530): Started retrieving bucket cache from file 2024-11-26T07:53:48,766 WARN [Thread-118 {}] bucket.BucketCache(1533): Persistence file missing! It's ok if it's first run after enabling persistent cache. 2024-11-26T07:53:48,767 INFO [Time-limited test {}] bucket.BucketCache(403): Started bucket cache; ioengine=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3acb3e4a-dea5-ea8d-f2d0-bb92c503e71b/bucket.cache, capacity=400 MB, blockSize=64 KB, writerThreadNum=3, writerQLen=64, persistencePath=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3acb3e4a-dea5-ea8d-f2d0-bb92c503e71b/bucket.persistence, bucketAllocator=org.apache.hadoop.hbase.io.hfile.bucket.BucketAllocator 2024-11-26T07:53:48,767 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-26T07:53:48,767 INFO [Thread-118 {}] bucket.BucketAllocator(343): Cache totalSize=418148352, buckets=199, bucket capacity=2101248=(4*525312)=(FEWEST_ITEMS_IN_BUCKET*(largest configured bucketcache size)) 2024-11-26T07:53:48,767 INFO [Thread-118 {}] bucket.BucketCache(413): Persistent bucket cache recovery from /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3acb3e4a-dea5-ea8d-f2d0-bb92c503e71b/bucket.persistence is complete. 2024-11-26T07:53:48,768 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:43013-0x101776be7e90001, quorum=127.0.0.1:64716, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-26T07:53:48,769 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:43013-0x101776be7e90001, quorum=127.0.0.1:64716, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-26T07:53:48,773 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=43013 2024-11-26T07:53:48,774 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=43013 2024-11-26T07:53:48,775 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=43013 2024-11-26T07:53:48,779 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=43013 2024-11-26T07:53:48,780 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=43013 2024-11-26T07:53:48,803 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/719ac8e87449:0 server-side Connection retries=45 2024-11-26T07:53:48,803 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-26T07:53:48,804 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-26T07:53:48,804 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-26T07:53:48,804 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-26T07:53:48,804 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-26T07:53:48,804 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-26T07:53:48,804 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-26T07:53:48,805 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:34097 2024-11-26T07:53:48,808 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:34097 connecting to ZooKeeper ensemble=127.0.0.1:64716 2024-11-26T07:53:48,808 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-26T07:53:48,808 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-26T07:53:48,824 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:340970x0, quorum=127.0.0.1:64716, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-26T07:53:48,825 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:34097-0x101776be7e90002 connected 2024-11-26T07:53:48,825 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:34097-0x101776be7e90002, quorum=127.0.0.1:64716, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-26T07:53:48,826 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-26T07:53:48,827 INFO [Time-limited test {}] bucket.FileIOEngine(99): Allocating cache 400 MB, on the path:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3acb3e4a-dea5-ea8d-f2d0-bb92c503e71b/bucket.cache 2024-11-26T07:53:48,827 INFO [Time-limited test {}] bucket.BucketCache(364): Instantiating BucketCache with acceptableFactor: 0.95, minFactor: 0.85, extraFreeFactor: 0.1, singleFactor: 0.25, multiFactor: 0.5, memoryFactor: 0.25, useStrongRef: false 2024-11-26T07:53:48,827 INFO [Time-limited test {}] bucket.BucketCache(478): Starting BucketCachePersisterThread 2024-11-26T07:53:48,827 INFO [Time-limited test {}] bucket.BucketCachePersister(38): BucketCachePersister started with interval: 100 2024-11-26T07:53:48,827 INFO [Thread-119 {}] bucket.BucketCache(1530): Started retrieving bucket cache from file 2024-11-26T07:53:48,827 WARN [Thread-119 {}] bucket.BucketCache(1533): Persistence file missing! It's ok if it's first run after enabling persistent cache. 2024-11-26T07:53:48,827 INFO [Time-limited test {}] bucket.BucketCache(403): Started bucket cache; ioengine=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3acb3e4a-dea5-ea8d-f2d0-bb92c503e71b/bucket.cache, capacity=400 MB, blockSize=64 KB, writerThreadNum=3, writerQLen=64, persistencePath=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3acb3e4a-dea5-ea8d-f2d0-bb92c503e71b/bucket.persistence, bucketAllocator=org.apache.hadoop.hbase.io.hfile.bucket.BucketAllocator 2024-11-26T07:53:48,828 INFO [Thread-119 {}] bucket.BucketAllocator(343): Cache totalSize=418148352, buckets=199, bucket capacity=2101248=(4*525312)=(FEWEST_ITEMS_IN_BUCKET*(largest configured bucketcache size)) 2024-11-26T07:53:48,828 INFO [Thread-119 {}] bucket.BucketCache(413): Persistent bucket cache recovery from /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3acb3e4a-dea5-ea8d-f2d0-bb92c503e71b/bucket.persistence is complete. 2024-11-26T07:53:48,828 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-26T07:53:48,829 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:34097-0x101776be7e90002, quorum=127.0.0.1:64716, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-26T07:53:48,831 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:34097-0x101776be7e90002, quorum=127.0.0.1:64716, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-26T07:53:48,838 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=34097 2024-11-26T07:53:48,840 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=34097 2024-11-26T07:53:48,841 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=34097 2024-11-26T07:53:48,842 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=34097 2024-11-26T07:53:48,847 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=34097 2024-11-26T07:53:48,862 DEBUG [M:0;719ac8e87449:36471 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;719ac8e87449:36471 2024-11-26T07:53:48,862 INFO [master/719ac8e87449:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/719ac8e87449,36471,1732607628597 2024-11-26T07:53:48,872 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36471-0x101776be7e90000, quorum=127.0.0.1:64716, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-26T07:53:48,872 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34097-0x101776be7e90002, quorum=127.0.0.1:64716, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-26T07:53:48,875 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43013-0x101776be7e90001, quorum=127.0.0.1:64716, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-26T07:53:48,876 DEBUG [master/719ac8e87449:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:36471-0x101776be7e90000, quorum=127.0.0.1:64716, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/719ac8e87449,36471,1732607628597 2024-11-26T07:53:48,882 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43013-0x101776be7e90001, quorum=127.0.0.1:64716, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-26T07:53:48,883 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34097-0x101776be7e90002, quorum=127.0.0.1:64716, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-26T07:53:48,883 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34097-0x101776be7e90002, quorum=127.0.0.1:64716, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-26T07:53:48,883 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43013-0x101776be7e90001, quorum=127.0.0.1:64716, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-26T07:53:48,884 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36471-0x101776be7e90000, quorum=127.0.0.1:64716, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-26T07:53:48,885 DEBUG [master/719ac8e87449:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:36471-0x101776be7e90000, quorum=127.0.0.1:64716, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-11-26T07:53:48,886 INFO [master/719ac8e87449:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/719ac8e87449,36471,1732607628597 from backup master directory 2024-11-26T07:53:48,897 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43013-0x101776be7e90001, quorum=127.0.0.1:64716, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-26T07:53:48,897 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36471-0x101776be7e90000, quorum=127.0.0.1:64716, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/719ac8e87449,36471,1732607628597 2024-11-26T07:53:48,897 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36471-0x101776be7e90000, quorum=127.0.0.1:64716, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-26T07:53:48,897 WARN [master/719ac8e87449:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-26T07:53:48,898 INFO [master/719ac8e87449:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=719ac8e87449,36471,1732607628597 2024-11-26T07:53:48,898 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34097-0x101776be7e90002, quorum=127.0.0.1:64716, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-26T07:53:48,899 DEBUG [master/719ac8e87449:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/480cd986-aef2-fe93-7892-18b5e497e91c/293d53df-6485-edb9-9b39-cf743ea8b2f2/hbase.id] with ID: 0bd26192-d019-4cda-81e6-944fe4f54fbe 2024-11-26T07:53:48,899 DEBUG [master/719ac8e87449:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/480cd986-aef2-fe93-7892-18b5e497e91c/293d53df-6485-edb9-9b39-cf743ea8b2f2/.tmp/hbase.id 2024-11-26T07:53:48,903 DEBUG [master/719ac8e87449:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/480cd986-aef2-fe93-7892-18b5e497e91c/293d53df-6485-edb9-9b39-cf743ea8b2f2/.tmp/hbase.id]:[file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/480cd986-aef2-fe93-7892-18b5e497e91c/293d53df-6485-edb9-9b39-cf743ea8b2f2/hbase.id] 2024-11-26T07:53:48,961 DEBUG [master/719ac8e87449:0:becomeActiveMaster {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-26T07:53:48,962 INFO [master/719ac8e87449:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem. 2024-11-26T07:53:48,964 INFO [master/719ac8e87449:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 2ms. 2024-11-26T07:53:48,982 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43013-0x101776be7e90001, quorum=127.0.0.1:64716, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-26T07:53:48,982 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36471-0x101776be7e90000, quorum=127.0.0.1:64716, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-26T07:53:48,982 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34097-0x101776be7e90002, quorum=127.0.0.1:64716, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-26T07:53:48,989 INFO [master/719ac8e87449:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-26T07:53:48,991 INFO [master/719ac8e87449:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-11-26T07:53:48,991 INFO [master/719ac8e87449:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-26T07:53:49,010 INFO [master/719ac8e87449:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/480cd986-aef2-fe93-7892-18b5e497e91c/293d53df-6485-edb9-9b39-cf743ea8b2f2/MasterData/data/master/store 2024-11-26T07:53:49,018 DEBUG [master/719ac8e87449:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-26T07:53:49,018 DEBUG [master/719ac8e87449:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-26T07:53:49,018 INFO [master/719ac8e87449:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-26T07:53:49,018 DEBUG [master/719ac8e87449:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-26T07:53:49,018 DEBUG [master/719ac8e87449:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-26T07:53:49,018 DEBUG [master/719ac8e87449:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-26T07:53:49,018 INFO [master/719ac8e87449:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-26T07:53:49,018 DEBUG [master/719ac8e87449:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1732607629018Disabling compacts and flushes for region at 1732607629018Disabling writes for close at 1732607629018Writing region close event to WAL at 1732607629018Closed at 1732607629018 2024-11-26T07:53:49,018 WARN [master/719ac8e87449:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/480cd986-aef2-fe93-7892-18b5e497e91c/293d53df-6485-edb9-9b39-cf743ea8b2f2/MasterData/data/master/store/.initializing 2024-11-26T07:53:49,018 DEBUG [master/719ac8e87449:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/480cd986-aef2-fe93-7892-18b5e497e91c/293d53df-6485-edb9-9b39-cf743ea8b2f2/MasterData/WALs/719ac8e87449,36471,1732607628597 2024-11-26T07:53:49,026 INFO [master/719ac8e87449:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-26T07:53:49,030 INFO [master/719ac8e87449:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=719ac8e87449%2C36471%2C1732607628597, suffix=, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/480cd986-aef2-fe93-7892-18b5e497e91c/293d53df-6485-edb9-9b39-cf743ea8b2f2/MasterData/WALs/719ac8e87449,36471,1732607628597, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/480cd986-aef2-fe93-7892-18b5e497e91c/293d53df-6485-edb9-9b39-cf743ea8b2f2/MasterData/oldWALs, maxLogs=10 2024-11-26T07:53:49,051 INFO [master/719ac8e87449:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/480cd986-aef2-fe93-7892-18b5e497e91c/293d53df-6485-edb9-9b39-cf743ea8b2f2/MasterData/WALs/719ac8e87449,36471,1732607628597/719ac8e87449%2C36471%2C1732607628597.1732607629030 2024-11-26T07:53:49,052 DEBUG [master/719ac8e87449:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-11-26T07:53:49,052 DEBUG [master/719ac8e87449:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-11-26T07:53:49,052 DEBUG [master/719ac8e87449:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-26T07:53:49,052 DEBUG [master/719ac8e87449:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-11-26T07:53:49,052 DEBUG [master/719ac8e87449:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-11-26T07:53:49,053 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-11-26T07:53:49,059 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-11-26T07:53:49,059 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T07:53:49,059 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-26T07:53:49,060 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-11-26T07:53:49,063 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-11-26T07:53:49,064 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T07:53:49,064 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-26T07:53:49,064 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-11-26T07:53:49,071 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-11-26T07:53:49,071 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T07:53:49,072 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-26T07:53:49,072 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-11-26T07:53:49,077 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-11-26T07:53:49,078 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T07:53:49,078 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-26T07:53:49,078 DEBUG [master/719ac8e87449:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682 2024-11-26T07:53:49,079 DEBUG [master/719ac8e87449:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/480cd986-aef2-fe93-7892-18b5e497e91c/293d53df-6485-edb9-9b39-cf743ea8b2f2/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-26T07:53:49,079 DEBUG [master/719ac8e87449:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/480cd986-aef2-fe93-7892-18b5e497e91c/293d53df-6485-edb9-9b39-cf743ea8b2f2/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-26T07:53:49,079 DEBUG [master/719ac8e87449:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682 2024-11-26T07:53:49,079 DEBUG [master/719ac8e87449:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682 2024-11-26T07:53:49,079 DEBUG [master/719ac8e87449:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-26T07:53:49,080 DEBUG [master/719ac8e87449:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-11-26T07:53:49,089 DEBUG [master/719ac8e87449:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/480cd986-aef2-fe93-7892-18b5e497e91c/293d53df-6485-edb9-9b39-cf743ea8b2f2/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-26T07:53:49,089 INFO [master/719ac8e87449:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=68242029, jitterRate=0.01688547432422638}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-26T07:53:49,091 DEBUG [master/719ac8e87449:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682: Writing region info on filesystem at 1732607629053Initializing all the Stores at 1732607629053Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732607629053Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732607629053Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732607629053Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732607629053Cleaning up temporary data from old regions at 1732607629079 (+26 ms)Region opened successfully at 1732607629090 (+11 ms) 2024-11-26T07:53:49,091 INFO [master/719ac8e87449:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-11-26T07:53:49,098 DEBUG [master/719ac8e87449:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2a8592f8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=719ac8e87449/172.17.0.2:0 2024-11-26T07:53:49,099 INFO [master/719ac8e87449:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating... 2024-11-26T07:53:49,099 INFO [master/719ac8e87449:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5 2024-11-26T07:53:49,099 INFO [master/719ac8e87449:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50 2024-11-26T07:53:49,100 INFO [master/719ac8e87449:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-26T07:53:49,100 INFO [master/719ac8e87449:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-26T07:53:49,101 INFO [master/719ac8e87449:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 0 msec 2024-11-26T07:53:49,101 INFO [master/719ac8e87449:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-11-26T07:53:49,103 INFO [master/719ac8e87449:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2024-11-26T07:53:49,105 DEBUG [master/719ac8e87449:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:36471-0x101776be7e90000, quorum=127.0.0.1:64716, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error) 2024-11-26T07:53:49,141 DEBUG [master/719ac8e87449:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false 2024-11-26T07:53:49,141 INFO [master/719ac8e87449:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2024-11-26T07:53:49,143 DEBUG [master/719ac8e87449:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:36471-0x101776be7e90000, quorum=127.0.0.1:64716, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error) 2024-11-26T07:53:49,149 DEBUG [master/719ac8e87449:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false 2024-11-26T07:53:49,150 INFO [master/719ac8e87449:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2024-11-26T07:53:49,152 DEBUG [master/719ac8e87449:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:36471-0x101776be7e90000, quorum=127.0.0.1:64716, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error) 2024-11-26T07:53:49,163 DEBUG [master/719ac8e87449:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false 2024-11-26T07:53:49,165 DEBUG [master/719ac8e87449:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:36471-0x101776be7e90000, quorum=127.0.0.1:64716, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error) 2024-11-26T07:53:49,174 DEBUG [master/719ac8e87449:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false 2024-11-26T07:53:49,178 DEBUG [master/719ac8e87449:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:36471-0x101776be7e90000, quorum=127.0.0.1:64716, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error) 2024-11-26T07:53:49,188 DEBUG [master/719ac8e87449:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false 2024-11-26T07:53:49,199 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34097-0x101776be7e90002, quorum=127.0.0.1:64716, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-26T07:53:49,199 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36471-0x101776be7e90000, quorum=127.0.0.1:64716, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-26T07:53:49,199 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34097-0x101776be7e90002, quorum=127.0.0.1:64716, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-26T07:53:49,199 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36471-0x101776be7e90000, quorum=127.0.0.1:64716, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-26T07:53:49,200 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43013-0x101776be7e90001, quorum=127.0.0.1:64716, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-26T07:53:49,200 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43013-0x101776be7e90001, quorum=127.0.0.1:64716, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-26T07:53:49,201 INFO [master/719ac8e87449:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=719ac8e87449,36471,1732607628597, sessionid=0x101776be7e90000, setting cluster-up flag (Was=false) 2024-11-26T07:53:49,216 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43013-0x101776be7e90001, quorum=127.0.0.1:64716, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-26T07:53:49,216 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34097-0x101776be7e90002, quorum=127.0.0.1:64716, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-26T07:53:49,216 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36471-0x101776be7e90000, quorum=127.0.0.1:64716, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-26T07:53:49,241 DEBUG [master/719ac8e87449:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort 2024-11-26T07:53:49,243 DEBUG [master/719ac8e87449:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=719ac8e87449,36471,1732607628597 2024-11-26T07:53:49,257 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34097-0x101776be7e90002, quorum=127.0.0.1:64716, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-26T07:53:49,257 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43013-0x101776be7e90001, quorum=127.0.0.1:64716, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-26T07:53:49,257 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36471-0x101776be7e90000, quorum=127.0.0.1:64716, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-26T07:53:49,282 DEBUG [master/719ac8e87449:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort 2024-11-26T07:53:49,285 DEBUG [master/719ac8e87449:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=719ac8e87449,36471,1732607628597 2024-11-26T07:53:49,285 INFO [master/719ac8e87449:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/480cd986-aef2-fe93-7892-18b5e497e91c/293d53df-6485-edb9-9b39-cf743ea8b2f2/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again 2024-11-26T07:53:49,287 DEBUG [master/719ac8e87449:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-26T07:53:49,288 INFO [master/719ac8e87449:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2 2024-11-26T07:53:49,288 INFO [master/719ac8e87449:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2024-11-26T07:53:49,288 DEBUG [master/719ac8e87449:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: 719ac8e87449,36471,1732607628597 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2024-11-26T07:53:49,290 DEBUG [master/719ac8e87449:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/719ac8e87449:0, corePoolSize=5, maxPoolSize=5 2024-11-26T07:53:49,290 DEBUG [master/719ac8e87449:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/719ac8e87449:0, corePoolSize=5, maxPoolSize=5 2024-11-26T07:53:49,291 DEBUG [master/719ac8e87449:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/719ac8e87449:0, corePoolSize=5, maxPoolSize=5 2024-11-26T07:53:49,291 DEBUG [master/719ac8e87449:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/719ac8e87449:0, corePoolSize=5, maxPoolSize=5 2024-11-26T07:53:49,291 DEBUG [master/719ac8e87449:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/719ac8e87449:0, corePoolSize=10, maxPoolSize=10 2024-11-26T07:53:49,291 DEBUG [master/719ac8e87449:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/719ac8e87449:0, corePoolSize=1, maxPoolSize=1 2024-11-26T07:53:49,291 DEBUG [master/719ac8e87449:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/719ac8e87449:0, corePoolSize=2, maxPoolSize=2 2024-11-26T07:53:49,291 DEBUG [master/719ac8e87449:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/719ac8e87449:0, corePoolSize=1, maxPoolSize=1 2024-11-26T07:53:49,296 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-26T07:53:49,297 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region 2024-11-26T07:53:49,297 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T07:53:49,298 INFO [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-11-26T07:53:49,309 INFO [master/719ac8e87449:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732607659309 2024-11-26T07:53:49,309 INFO [master/719ac8e87449:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2024-11-26T07:53:49,309 INFO [master/719ac8e87449:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2024-11-26T07:53:49,309 INFO [master/719ac8e87449:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2024-11-26T07:53:49,309 INFO [master/719ac8e87449:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2024-11-26T07:53:49,310 INFO [master/719ac8e87449:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2024-11-26T07:53:49,310 INFO [master/719ac8e87449:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2024-11-26T07:53:49,311 INFO [master/719ac8e87449:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-26T07:53:49,312 INFO [master/719ac8e87449:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2024-11-26T07:53:49,312 INFO [master/719ac8e87449:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2024-11-26T07:53:49,312 INFO [master/719ac8e87449:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2024-11-26T07:53:49,313 INFO [master/719ac8e87449:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2024-11-26T07:53:49,313 INFO [master/719ac8e87449:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2024-11-26T07:53:49,314 DEBUG [master/719ac8e87449:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/719ac8e87449:0:becomeActiveMaster-HFileCleaner.large.0-1732607629313,5,FailOnTimeoutGroup] 2024-11-26T07:53:49,314 DEBUG [master/719ac8e87449:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/719ac8e87449:0:becomeActiveMaster-HFileCleaner.small.0-1732607629314,5,FailOnTimeoutGroup] 2024-11-26T07:53:49,314 INFO [master/719ac8e87449:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-26T07:53:49,314 INFO [master/719ac8e87449:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2024-11-26T07:53:49,314 INFO [master/719ac8e87449:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2024-11-26T07:53:49,314 INFO [master/719ac8e87449:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2024-11-26T07:53:49,340 INFO [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/480cd986-aef2-fe93-7892-18b5e497e91c/293d53df-6485-edb9-9b39-cf743ea8b2f2/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321 2024-11-26T07:53:49,341 INFO [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/480cd986-aef2-fe93-7892-18b5e497e91c/293d53df-6485-edb9-9b39-cf743ea8b2f2 2024-11-26T07:53:49,350 INFO [RS:0;719ac8e87449:43013 {}] regionserver.HRegionServer(746): ClusterId : 0bd26192-d019-4cda-81e6-944fe4f54fbe 2024-11-26T07:53:49,350 DEBUG [RS:0;719ac8e87449:43013 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-26T07:53:49,351 INFO [RS:1;719ac8e87449:34097 {}] regionserver.HRegionServer(746): ClusterId : 0bd26192-d019-4cda-81e6-944fe4f54fbe 2024-11-26T07:53:49,351 DEBUG [RS:1;719ac8e87449:34097 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-26T07:53:49,356 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-26T07:53:49,358 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family info of region 1588230740 2024-11-26T07:53:49,363 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-26T07:53:49,363 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T07:53:49,363 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-26T07:53:49,364 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family ns of region 1588230740 2024-11-26T07:53:49,371 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-26T07:53:49,371 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T07:53:49,371 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-26T07:53:49,371 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family rep_barrier of region 1588230740 2024-11-26T07:53:49,384 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-26T07:53:49,384 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T07:53:49,385 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-26T07:53:49,385 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family table of region 1588230740 2024-11-26T07:53:49,390 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-26T07:53:49,390 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T07:53:49,394 DEBUG [RS:0;719ac8e87449:43013 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-26T07:53:49,394 DEBUG [RS:0;719ac8e87449:43013 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-26T07:53:49,395 DEBUG [RS:1;719ac8e87449:34097 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-26T07:53:49,395 DEBUG [RS:1;719ac8e87449:34097 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-26T07:53:49,395 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-26T07:53:49,395 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-26T07:53:49,396 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/480cd986-aef2-fe93-7892-18b5e497e91c/293d53df-6485-edb9-9b39-cf743ea8b2f2/data/hbase/meta/1588230740 2024-11-26T07:53:49,396 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/480cd986-aef2-fe93-7892-18b5e497e91c/293d53df-6485-edb9-9b39-cf743ea8b2f2/data/hbase/meta/1588230740 2024-11-26T07:53:49,397 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-26T07:53:49,397 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-26T07:53:49,397 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-26T07:53:49,397 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-26T07:53:49,410 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/480cd986-aef2-fe93-7892-18b5e497e91c/293d53df-6485-edb9-9b39-cf743ea8b2f2/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-26T07:53:49,412 INFO [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=71585997, jitterRate=0.06671448051929474}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-26T07:53:49,415 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740: Writing region info on filesystem at 1732607629356Initializing all the Stores at 1732607629357 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732607629357Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732607629358 (+1 ms)Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732607629358Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732607629358Cleaning up temporary data from old regions at 1732607629397 (+39 ms)Region opened successfully at 1732607629415 (+18 ms) 2024-11-26T07:53:49,417 DEBUG [RS:0;719ac8e87449:43013 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-26T07:53:49,417 DEBUG [RS:1;719ac8e87449:34097 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-26T07:53:49,418 DEBUG [RS:1;719ac8e87449:34097 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@42bf5bb8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=719ac8e87449/172.17.0.2:0 2024-11-26T07:53:49,418 DEBUG [RS:0;719ac8e87449:43013 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@416338d7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=719ac8e87449/172.17.0.2:0 2024-11-26T07:53:49,418 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-26T07:53:49,419 INFO [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-26T07:53:49,419 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-26T07:53:49,419 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-26T07:53:49,419 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-26T07:53:49,423 INFO [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-26T07:53:49,423 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1732607629415Disabling compacts and flushes for region at 1732607629415Disabling writes for close at 1732607629419 (+4 ms)Writing region close event to WAL at 1732607629423 (+4 ms)Closed at 1732607629423 2024-11-26T07:53:49,424 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-26T07:53:49,424 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta 2024-11-26T07:53:49,425 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-11-26T07:53:49,427 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-26T07:53:49,428 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2024-11-26T07:53:49,436 DEBUG [RS:1;719ac8e87449:34097 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:1;719ac8e87449:34097 2024-11-26T07:53:49,436 INFO [RS:1;719ac8e87449:34097 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-26T07:53:49,436 DEBUG [RS:0;719ac8e87449:43013 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;719ac8e87449:43013 2024-11-26T07:53:49,436 INFO [RS:1;719ac8e87449:34097 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-26T07:53:49,436 DEBUG [RS:1;719ac8e87449:34097 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-26T07:53:49,436 INFO [RS:0;719ac8e87449:43013 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-26T07:53:49,436 INFO [RS:0;719ac8e87449:43013 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-26T07:53:49,436 DEBUG [RS:0;719ac8e87449:43013 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-26T07:53:49,437 INFO [RS:1;719ac8e87449:34097 {}] regionserver.HRegionServer(2659): reportForDuty to master=719ac8e87449,36471,1732607628597 with port=34097, startcode=1732607628803 2024-11-26T07:53:49,437 INFO [RS:0;719ac8e87449:43013 {}] regionserver.HRegionServer(2659): reportForDuty to master=719ac8e87449,36471,1732607628597 with port=43013, startcode=1732607628743 2024-11-26T07:53:49,438 DEBUG [RS:0;719ac8e87449:43013 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-26T07:53:49,438 DEBUG [RS:1;719ac8e87449:34097 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-26T07:53:49,443 INFO [HMaster-EventLoopGroup-6-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56403, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=RegionServerStatusService 2024-11-26T07:53:49,445 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=36471 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 719ac8e87449,43013,1732607628743 2024-11-26T07:53:49,445 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=36471 {}] master.ServerManager(517): Registering regionserver=719ac8e87449,43013,1732607628743 2024-11-26T07:53:49,450 DEBUG [RS:0;719ac8e87449:43013 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/480cd986-aef2-fe93-7892-18b5e497e91c/293d53df-6485-edb9-9b39-cf743ea8b2f2 2024-11-26T07:53:49,450 DEBUG [RS:0;719ac8e87449:43013 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=file:/ 2024-11-26T07:53:49,450 DEBUG [RS:0;719ac8e87449:43013 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-26T07:53:49,453 INFO [HMaster-EventLoopGroup-6-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45429, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=RegionServerStatusService 2024-11-26T07:53:49,454 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=36471 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 719ac8e87449,34097,1732607628803 2024-11-26T07:53:49,454 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=36471 {}] master.ServerManager(517): Registering regionserver=719ac8e87449,34097,1732607628803 2024-11-26T07:53:49,456 DEBUG [RS:1;719ac8e87449:34097 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/480cd986-aef2-fe93-7892-18b5e497e91c/293d53df-6485-edb9-9b39-cf743ea8b2f2 2024-11-26T07:53:49,456 DEBUG [RS:1;719ac8e87449:34097 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=file:/ 2024-11-26T07:53:49,456 DEBUG [RS:1;719ac8e87449:34097 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-26T07:53:49,457 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36471-0x101776be7e90000, quorum=127.0.0.1:64716, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-26T07:53:49,484 DEBUG [RS:0;719ac8e87449:43013 {}] zookeeper.ZKUtil(111): regionserver:43013-0x101776be7e90001, quorum=127.0.0.1:64716, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/719ac8e87449,43013,1732607628743 2024-11-26T07:53:49,484 WARN [RS:0;719ac8e87449:43013 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-26T07:53:49,484 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [719ac8e87449,43013,1732607628743] 2024-11-26T07:53:49,484 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [719ac8e87449,34097,1732607628803] 2024-11-26T07:53:49,484 INFO [RS:0;719ac8e87449:43013 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-26T07:53:49,484 DEBUG [RS:1;719ac8e87449:34097 {}] zookeeper.ZKUtil(111): regionserver:34097-0x101776be7e90002, quorum=127.0.0.1:64716, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/719ac8e87449,34097,1732607628803 2024-11-26T07:53:49,484 DEBUG [RS:0;719ac8e87449:43013 {}] regionserver.HRegionServer(1793): logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/480cd986-aef2-fe93-7892-18b5e497e91c/293d53df-6485-edb9-9b39-cf743ea8b2f2/WALs/719ac8e87449,43013,1732607628743 2024-11-26T07:53:49,484 WARN [RS:1;719ac8e87449:34097 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-26T07:53:49,484 INFO [RS:1;719ac8e87449:34097 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-26T07:53:49,484 DEBUG [RS:1;719ac8e87449:34097 {}] regionserver.HRegionServer(1793): logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/480cd986-aef2-fe93-7892-18b5e497e91c/293d53df-6485-edb9-9b39-cf743ea8b2f2/WALs/719ac8e87449,34097,1732607628803 2024-11-26T07:53:49,492 INFO [RS:1;719ac8e87449:34097 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-26T07:53:49,492 INFO [RS:0;719ac8e87449:43013 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-26T07:53:49,495 INFO [RS:1;719ac8e87449:34097 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-26T07:53:49,500 INFO [RS:1;719ac8e87449:34097 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-26T07:53:49,500 INFO [RS:1;719ac8e87449:34097 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-26T07:53:49,506 INFO [RS:0;719ac8e87449:43013 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-26T07:53:49,507 INFO [RS:1;719ac8e87449:34097 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-26T07:53:49,507 INFO [RS:1;719ac8e87449:34097 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-26T07:53:49,507 INFO [RS:1;719ac8e87449:34097 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-26T07:53:49,508 DEBUG [RS:1;719ac8e87449:34097 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/719ac8e87449:0, corePoolSize=1, maxPoolSize=1 2024-11-26T07:53:49,508 DEBUG [RS:1;719ac8e87449:34097 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/719ac8e87449:0, corePoolSize=1, maxPoolSize=1 2024-11-26T07:53:49,508 DEBUG [RS:1;719ac8e87449:34097 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/719ac8e87449:0, corePoolSize=1, maxPoolSize=1 2024-11-26T07:53:49,508 DEBUG [RS:1;719ac8e87449:34097 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/719ac8e87449:0, corePoolSize=1, maxPoolSize=1 2024-11-26T07:53:49,508 DEBUG [RS:1;719ac8e87449:34097 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/719ac8e87449:0, corePoolSize=1, maxPoolSize=1 2024-11-26T07:53:49,508 DEBUG [RS:1;719ac8e87449:34097 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/719ac8e87449:0, corePoolSize=2, maxPoolSize=2 2024-11-26T07:53:49,508 DEBUG [RS:1;719ac8e87449:34097 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/719ac8e87449:0, corePoolSize=1, maxPoolSize=1 2024-11-26T07:53:49,508 DEBUG [RS:1;719ac8e87449:34097 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/719ac8e87449:0, corePoolSize=1, maxPoolSize=1 2024-11-26T07:53:49,509 DEBUG [RS:1;719ac8e87449:34097 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/719ac8e87449:0, corePoolSize=1, maxPoolSize=1 2024-11-26T07:53:49,509 DEBUG [RS:1;719ac8e87449:34097 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/719ac8e87449:0, corePoolSize=1, maxPoolSize=1 2024-11-26T07:53:49,509 DEBUG [RS:1;719ac8e87449:34097 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/719ac8e87449:0, corePoolSize=1, maxPoolSize=1 2024-11-26T07:53:49,509 DEBUG [RS:1;719ac8e87449:34097 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/719ac8e87449:0, corePoolSize=1, maxPoolSize=1 2024-11-26T07:53:49,509 DEBUG [RS:1;719ac8e87449:34097 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/719ac8e87449:0, corePoolSize=3, maxPoolSize=3 2024-11-26T07:53:49,509 DEBUG [RS:1;719ac8e87449:34097 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/719ac8e87449:0, corePoolSize=3, maxPoolSize=3 2024-11-26T07:53:49,510 INFO [RS:0;719ac8e87449:43013 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-26T07:53:49,510 INFO [RS:0;719ac8e87449:43013 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-26T07:53:49,510 INFO [RS:0;719ac8e87449:43013 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-26T07:53:49,511 INFO [RS:0;719ac8e87449:43013 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-26T07:53:49,511 INFO [RS:0;719ac8e87449:43013 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-26T07:53:49,511 DEBUG [RS:0;719ac8e87449:43013 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/719ac8e87449:0, corePoolSize=1, maxPoolSize=1 2024-11-26T07:53:49,511 DEBUG [RS:0;719ac8e87449:43013 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/719ac8e87449:0, corePoolSize=1, maxPoolSize=1 2024-11-26T07:53:49,511 DEBUG [RS:0;719ac8e87449:43013 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/719ac8e87449:0, corePoolSize=1, maxPoolSize=1 2024-11-26T07:53:49,511 DEBUG [RS:0;719ac8e87449:43013 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/719ac8e87449:0, corePoolSize=1, maxPoolSize=1 2024-11-26T07:53:49,511 DEBUG [RS:0;719ac8e87449:43013 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/719ac8e87449:0, corePoolSize=1, maxPoolSize=1 2024-11-26T07:53:49,512 DEBUG [RS:0;719ac8e87449:43013 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/719ac8e87449:0, corePoolSize=2, maxPoolSize=2 2024-11-26T07:53:49,512 DEBUG [RS:0;719ac8e87449:43013 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/719ac8e87449:0, corePoolSize=1, maxPoolSize=1 2024-11-26T07:53:49,512 DEBUG [RS:0;719ac8e87449:43013 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/719ac8e87449:0, corePoolSize=1, maxPoolSize=1 2024-11-26T07:53:49,512 DEBUG [RS:0;719ac8e87449:43013 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/719ac8e87449:0, corePoolSize=1, maxPoolSize=1 2024-11-26T07:53:49,512 DEBUG [RS:0;719ac8e87449:43013 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/719ac8e87449:0, corePoolSize=1, maxPoolSize=1 2024-11-26T07:53:49,512 DEBUG [RS:0;719ac8e87449:43013 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/719ac8e87449:0, corePoolSize=1, maxPoolSize=1 2024-11-26T07:53:49,512 DEBUG [RS:0;719ac8e87449:43013 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/719ac8e87449:0, corePoolSize=1, maxPoolSize=1 2024-11-26T07:53:49,512 DEBUG [RS:0;719ac8e87449:43013 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/719ac8e87449:0, corePoolSize=3, maxPoolSize=3 2024-11-26T07:53:49,512 DEBUG [RS:0;719ac8e87449:43013 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/719ac8e87449:0, corePoolSize=3, maxPoolSize=3 2024-11-26T07:53:49,515 INFO [RS:1;719ac8e87449:34097 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-26T07:53:49,515 INFO [RS:1;719ac8e87449:34097 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-26T07:53:49,515 INFO [RS:1;719ac8e87449:34097 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-26T07:53:49,516 INFO [RS:1;719ac8e87449:34097 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-26T07:53:49,516 INFO [RS:1;719ac8e87449:34097 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-26T07:53:49,516 INFO [RS:1;719ac8e87449:34097 {}] hbase.ChoreService(168): Chore ScheduledChore name=719ac8e87449,34097,1732607628803-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-26T07:53:49,518 INFO [RS:0;719ac8e87449:43013 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-26T07:53:49,518 INFO [RS:0;719ac8e87449:43013 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-26T07:53:49,518 INFO [RS:0;719ac8e87449:43013 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-26T07:53:49,518 INFO [RS:0;719ac8e87449:43013 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-26T07:53:49,518 INFO [RS:0;719ac8e87449:43013 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-26T07:53:49,518 INFO [RS:0;719ac8e87449:43013 {}] hbase.ChoreService(168): Chore ScheduledChore name=719ac8e87449,43013,1732607628743-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-26T07:53:49,538 INFO [RS:1;719ac8e87449:34097 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-26T07:53:49,539 INFO [RS:1;719ac8e87449:34097 {}] hbase.ChoreService(168): Chore ScheduledChore name=719ac8e87449,34097,1732607628803-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-26T07:53:49,539 INFO [RS:1;719ac8e87449:34097 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-26T07:53:49,539 INFO [RS:1;719ac8e87449:34097 {}] regionserver.Replication(171): 719ac8e87449,34097,1732607628803 started 2024-11-26T07:53:49,540 INFO [RS:0;719ac8e87449:43013 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-26T07:53:49,540 INFO [RS:0;719ac8e87449:43013 {}] hbase.ChoreService(168): Chore ScheduledChore name=719ac8e87449,43013,1732607628743-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-26T07:53:49,540 INFO [RS:0;719ac8e87449:43013 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-26T07:53:49,541 INFO [RS:0;719ac8e87449:43013 {}] regionserver.Replication(171): 719ac8e87449,43013,1732607628743 started 2024-11-26T07:53:49,556 INFO [RS:1;719ac8e87449:34097 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-26T07:53:49,556 INFO [RS:1;719ac8e87449:34097 {}] regionserver.HRegionServer(1482): Serving as 719ac8e87449,34097,1732607628803, RpcServer on 719ac8e87449/172.17.0.2:34097, sessionid=0x101776be7e90002 2024-11-26T07:53:49,557 DEBUG [RS:1;719ac8e87449:34097 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-26T07:53:49,557 DEBUG [RS:1;719ac8e87449:34097 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 719ac8e87449,34097,1732607628803 2024-11-26T07:53:49,557 DEBUG [RS:1;719ac8e87449:34097 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '719ac8e87449,34097,1732607628803' 2024-11-26T07:53:49,557 DEBUG [RS:1;719ac8e87449:34097 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-26T07:53:49,558 DEBUG [RS:1;719ac8e87449:34097 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-26T07:53:49,558 DEBUG [RS:1;719ac8e87449:34097 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-26T07:53:49,558 DEBUG [RS:1;719ac8e87449:34097 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-26T07:53:49,558 DEBUG [RS:1;719ac8e87449:34097 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 719ac8e87449,34097,1732607628803 2024-11-26T07:53:49,558 DEBUG [RS:1;719ac8e87449:34097 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '719ac8e87449,34097,1732607628803' 2024-11-26T07:53:49,558 DEBUG [RS:1;719ac8e87449:34097 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-26T07:53:49,559 DEBUG [RS:1;719ac8e87449:34097 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-26T07:53:49,559 DEBUG [RS:1;719ac8e87449:34097 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-26T07:53:49,559 INFO [RS:1;719ac8e87449:34097 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-26T07:53:49,559 INFO [RS:1;719ac8e87449:34097 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-26T07:53:49,562 INFO [RS:0;719ac8e87449:43013 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-26T07:53:49,562 INFO [RS:0;719ac8e87449:43013 {}] regionserver.HRegionServer(1482): Serving as 719ac8e87449,43013,1732607628743, RpcServer on 719ac8e87449/172.17.0.2:43013, sessionid=0x101776be7e90001 2024-11-26T07:53:49,562 DEBUG [RS:0;719ac8e87449:43013 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-26T07:53:49,562 DEBUG [RS:0;719ac8e87449:43013 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 719ac8e87449,43013,1732607628743 2024-11-26T07:53:49,562 DEBUG [RS:0;719ac8e87449:43013 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '719ac8e87449,43013,1732607628743' 2024-11-26T07:53:49,563 DEBUG [RS:0;719ac8e87449:43013 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-26T07:53:49,565 DEBUG [RS:0;719ac8e87449:43013 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-26T07:53:49,566 DEBUG [RS:0;719ac8e87449:43013 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-26T07:53:49,566 DEBUG [RS:0;719ac8e87449:43013 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-26T07:53:49,566 DEBUG [RS:0;719ac8e87449:43013 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 719ac8e87449,43013,1732607628743 2024-11-26T07:53:49,566 DEBUG [RS:0;719ac8e87449:43013 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '719ac8e87449,43013,1732607628743' 2024-11-26T07:53:49,566 DEBUG [RS:0;719ac8e87449:43013 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-26T07:53:49,567 DEBUG [RS:0;719ac8e87449:43013 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-26T07:53:49,567 DEBUG [RS:0;719ac8e87449:43013 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-26T07:53:49,567 INFO [RS:0;719ac8e87449:43013 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-26T07:53:49,567 INFO [RS:0;719ac8e87449:43013 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-26T07:53:49,578 WARN [719ac8e87449:36471 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-26T07:53:49,660 INFO [RS:1;719ac8e87449:34097 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-26T07:53:49,661 INFO [RS:1;719ac8e87449:34097 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=719ac8e87449%2C34097%2C1732607628803, suffix=, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/480cd986-aef2-fe93-7892-18b5e497e91c/293d53df-6485-edb9-9b39-cf743ea8b2f2/WALs/719ac8e87449,34097,1732607628803, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/480cd986-aef2-fe93-7892-18b5e497e91c/293d53df-6485-edb9-9b39-cf743ea8b2f2/oldWALs, maxLogs=55 2024-11-26T07:53:49,668 INFO [RS:0;719ac8e87449:43013 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-26T07:53:49,669 INFO [RS:0;719ac8e87449:43013 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=719ac8e87449%2C43013%2C1732607628743, suffix=, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/480cd986-aef2-fe93-7892-18b5e497e91c/293d53df-6485-edb9-9b39-cf743ea8b2f2/WALs/719ac8e87449,43013,1732607628743, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/480cd986-aef2-fe93-7892-18b5e497e91c/293d53df-6485-edb9-9b39-cf743ea8b2f2/oldWALs, maxLogs=55 2024-11-26T07:53:49,683 INFO [RS:1;719ac8e87449:34097 {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/480cd986-aef2-fe93-7892-18b5e497e91c/293d53df-6485-edb9-9b39-cf743ea8b2f2/WALs/719ac8e87449,34097,1732607628803/719ac8e87449%2C34097%2C1732607628803.1732607629662 2024-11-26T07:53:49,685 DEBUG [RS:1;719ac8e87449:34097 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-11-26T07:53:49,689 INFO [RS:0;719ac8e87449:43013 {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/480cd986-aef2-fe93-7892-18b5e497e91c/293d53df-6485-edb9-9b39-cf743ea8b2f2/WALs/719ac8e87449,43013,1732607628743/719ac8e87449%2C43013%2C1732607628743.1732607629670 2024-11-26T07:53:49,689 DEBUG [RS:0;719ac8e87449:43013 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-11-26T07:53:49,829 DEBUG [719ac8e87449:36471 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=2, allServersCount=2 2024-11-26T07:53:49,829 DEBUG [719ac8e87449:36471 {}] balancer.BalancerClusterState(204): Hosts are {719ac8e87449=0} racks are {/default-rack=0} 2024-11-26T07:53:49,830 DEBUG [719ac8e87449:36471 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-11-26T07:53:49,830 DEBUG [719ac8e87449:36471 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-11-26T07:53:49,830 DEBUG [719ac8e87449:36471 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-11-26T07:53:49,830 DEBUG [719ac8e87449:36471 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-11-26T07:53:49,830 INFO [719ac8e87449:36471 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-11-26T07:53:49,830 INFO [719ac8e87449:36471 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-11-26T07:53:49,830 DEBUG [719ac8e87449:36471 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-11-26T07:53:49,830 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=719ac8e87449,43013,1732607628743 2024-11-26T07:53:49,831 INFO [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 719ac8e87449,43013,1732607628743, state=OPENING 2024-11-26T07:53:49,864 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2024-11-26T07:53:49,874 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43013-0x101776be7e90001, quorum=127.0.0.1:64716, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-26T07:53:49,874 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34097-0x101776be7e90002, quorum=127.0.0.1:64716, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-26T07:53:49,874 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36471-0x101776be7e90000, quorum=127.0.0.1:64716, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-26T07:53:49,876 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-26T07:53:49,879 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-26T07:53:49,879 DEBUG [PEWorker-3 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-26T07:53:49,879 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=719ac8e87449,43013,1732607628743}] 2024-11-26T07:53:49,882 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-26T07:53:50,034 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-26T07:53:50,036 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-7-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47933, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-26T07:53:50,039 INFO [RS_OPEN_META-regionserver/719ac8e87449:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-11-26T07:53:50,039 INFO [RS_OPEN_META-regionserver/719ac8e87449:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-26T07:53:50,040 INFO [RS_OPEN_META-regionserver/719ac8e87449:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-11-26T07:53:50,041 INFO [RS_OPEN_META-regionserver/719ac8e87449:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=719ac8e87449%2C43013%2C1732607628743.meta, suffix=.meta, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/480cd986-aef2-fe93-7892-18b5e497e91c/293d53df-6485-edb9-9b39-cf743ea8b2f2/WALs/719ac8e87449,43013,1732607628743, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/480cd986-aef2-fe93-7892-18b5e497e91c/293d53df-6485-edb9-9b39-cf743ea8b2f2/oldWALs, maxLogs=55 2024-11-26T07:53:50,064 INFO [RS_OPEN_META-regionserver/719ac8e87449:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/480cd986-aef2-fe93-7892-18b5e497e91c/293d53df-6485-edb9-9b39-cf743ea8b2f2/WALs/719ac8e87449,43013,1732607628743/719ac8e87449%2C43013%2C1732607628743.meta.1732607630042.meta 2024-11-26T07:53:50,065 DEBUG [RS_OPEN_META-regionserver/719ac8e87449:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-11-26T07:53:50,066 DEBUG [RS_OPEN_META-regionserver/719ac8e87449:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-11-26T07:53:50,066 DEBUG [RS_OPEN_META-regionserver/719ac8e87449:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-11-26T07:53:50,066 DEBUG [RS_OPEN_META-regionserver/719ac8e87449:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-11-26T07:53:50,067 INFO [RS_OPEN_META-regionserver/719ac8e87449:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-11-26T07:53:50,067 DEBUG [RS_OPEN_META-regionserver/719ac8e87449:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-11-26T07:53:50,067 DEBUG [RS_OPEN_META-regionserver/719ac8e87449:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-26T07:53:50,067 DEBUG [RS_OPEN_META-regionserver/719ac8e87449:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-11-26T07:53:50,067 DEBUG [RS_OPEN_META-regionserver/719ac8e87449:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-11-26T07:53:50,068 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family info of region 1588230740 2024-11-26T07:53:50,069 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-26T07:53:50,069 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T07:53:50,070 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-26T07:53:50,070 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family ns of region 1588230740 2024-11-26T07:53:50,070 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-26T07:53:50,071 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T07:53:50,071 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-26T07:53:50,071 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family rep_barrier of region 1588230740 2024-11-26T07:53:50,072 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-26T07:53:50,072 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T07:53:50,072 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-26T07:53:50,072 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family table of region 1588230740 2024-11-26T07:53:50,073 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-26T07:53:50,073 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T07:53:50,073 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-26T07:53:50,073 DEBUG [RS_OPEN_META-regionserver/719ac8e87449:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-26T07:53:50,074 DEBUG [RS_OPEN_META-regionserver/719ac8e87449:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/480cd986-aef2-fe93-7892-18b5e497e91c/293d53df-6485-edb9-9b39-cf743ea8b2f2/data/hbase/meta/1588230740 2024-11-26T07:53:50,075 DEBUG [RS_OPEN_META-regionserver/719ac8e87449:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/480cd986-aef2-fe93-7892-18b5e497e91c/293d53df-6485-edb9-9b39-cf743ea8b2f2/data/hbase/meta/1588230740 2024-11-26T07:53:50,075 DEBUG [RS_OPEN_META-regionserver/719ac8e87449:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-26T07:53:50,075 DEBUG [RS_OPEN_META-regionserver/719ac8e87449:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-26T07:53:50,076 DEBUG [RS_OPEN_META-regionserver/719ac8e87449:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-26T07:53:50,077 DEBUG [RS_OPEN_META-regionserver/719ac8e87449:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-26T07:53:50,077 INFO [RS_OPEN_META-regionserver/719ac8e87449:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=67644305, jitterRate=0.007978692650794983}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-26T07:53:50,077 DEBUG [RS_OPEN_META-regionserver/719ac8e87449:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-11-26T07:53:50,079 DEBUG [RS_OPEN_META-regionserver/719ac8e87449:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1732607630067Writing region info on filesystem at 1732607630067Initializing all the Stores at 1732607630068 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732607630068Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732607630068Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732607630068Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732607630068Cleaning up temporary data from old regions at 1732607630075 (+7 ms)Running coprocessor post-open hooks at 1732607630077 (+2 ms)Region opened successfully at 1732607630079 (+2 ms) 2024-11-26T07:53:50,079 INFO [RS_OPEN_META-regionserver/719ac8e87449:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1732607630033 2024-11-26T07:53:50,083 DEBUG [RS_OPEN_META-regionserver/719ac8e87449:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-11-26T07:53:50,083 INFO [RS_OPEN_META-regionserver/719ac8e87449:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-11-26T07:53:50,083 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=719ac8e87449,43013,1732607628743 2024-11-26T07:53:50,084 INFO [PEWorker-4 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 719ac8e87449,43013,1732607628743, state=OPEN 2024-11-26T07:53:50,116 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34097-0x101776be7e90002, quorum=127.0.0.1:64716, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-26T07:53:50,116 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36471-0x101776be7e90000, quorum=127.0.0.1:64716, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-26T07:53:50,116 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43013-0x101776be7e90001, quorum=127.0.0.1:64716, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-26T07:53:50,116 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-26T07:53:50,116 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-26T07:53:50,116 DEBUG [PEWorker-4 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=719ac8e87449,43013,1732607628743 2024-11-26T07:53:50,116 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-26T07:53:50,120 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2 2024-11-26T07:53:50,120 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=719ac8e87449,43013,1732607628743 in 237 msec 2024-11-26T07:53:50,122 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1 2024-11-26T07:53:50,122 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 695 msec 2024-11-26T07:53:50,123 DEBUG [PEWorker-2 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-26T07:53:50,123 INFO [PEWorker-2 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces 2024-11-26T07:53:50,126 DEBUG [PEWorker-2 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-26T07:53:50,126 DEBUG [PEWorker-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=719ac8e87449,43013,1732607628743, seqNum=-1] 2024-11-26T07:53:50,127 DEBUG [PEWorker-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-26T07:53:50,129 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-7-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45123, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-26T07:53:50,135 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 847 msec 2024-11-26T07:53:50,135 INFO [master/719ac8e87449:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1732607630135, completionTime=-1 2024-11-26T07:53:50,135 INFO [master/719ac8e87449:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=2; waited=0ms, expected min=2 server(s), max=2 server(s), master is running 2024-11-26T07:53:50,135 DEBUG [master/719ac8e87449:0:becomeActiveMaster {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-26T07:53:50,138 INFO [master/719ac8e87449:0:becomeActiveMaster {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-11-26T07:53:50,138 INFO [master/719ac8e87449:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732607690138 2024-11-26T07:53:50,138 INFO [master/719ac8e87449:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732607750138 2024-11-26T07:53:50,138 INFO [master/719ac8e87449:0:becomeActiveMaster {}] assignment.AssignmentManager(1783): Joined the cluster in 2 msec 2024-11-26T07:53:50,139 DEBUG [master/719ac8e87449:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 1588230740 changed from -1.0 to 0.0, refreshing cache 2024-11-26T07:53:50,139 INFO [master/719ac8e87449:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=719ac8e87449,36471,1732607628597-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-26T07:53:50,140 INFO [master/719ac8e87449:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=719ac8e87449,36471,1732607628597-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-26T07:53:50,140 INFO [master/719ac8e87449:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=719ac8e87449,36471,1732607628597-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-26T07:53:50,140 INFO [master/719ac8e87449:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-719ac8e87449:36471, period=300000, unit=MILLISECONDS is enabled. 2024-11-26T07:53:50,140 INFO [master/719ac8e87449:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2024-11-26T07:53:50,143 DEBUG [master/719ac8e87449:0.Chore.1 {}] janitor.CatalogJanitor(180): 2024-11-26T07:53:50,146 INFO [master/719ac8e87449:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled. 2024-11-26T07:53:50,152 INFO [master/719ac8e87449:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 1.254sec 2024-11-26T07:53:50,153 INFO [master/719ac8e87449:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2024-11-26T07:53:50,153 INFO [master/719ac8e87449:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2024-11-26T07:53:50,153 INFO [master/719ac8e87449:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2024-11-26T07:53:50,153 INFO [master/719ac8e87449:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2024-11-26T07:53:50,153 INFO [master/719ac8e87449:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2024-11-26T07:53:50,153 INFO [master/719ac8e87449:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=719ac8e87449,36471,1732607628597-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-26T07:53:50,153 INFO [master/719ac8e87449:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=719ac8e87449,36471,1732607628597-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2024-11-26T07:53:50,154 DEBUG [master/719ac8e87449:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds 2024-11-26T07:53:50,154 INFO [master/719ac8e87449:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2024-11-26T07:53:50,154 INFO [master/719ac8e87449:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=719ac8e87449,36471,1732607628597-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-26T07:53:50,251 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@689405c2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T07:53:50,251 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 719ac8e87449,36471,-1 for getting cluster id 2024-11-26T07:53:50,251 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-26T07:53:50,253 DEBUG [HMaster-EventLoopGroup-6-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '0bd26192-d019-4cda-81e6-944fe4f54fbe' 2024-11-26T07:53:50,253 DEBUG [RPCClient-NioEventLoopGroup-5-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-26T07:53:50,253 DEBUG [RPCClient-NioEventLoopGroup-5-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "0bd26192-d019-4cda-81e6-944fe4f54fbe" 2024-11-26T07:53:50,254 DEBUG [RPCClient-NioEventLoopGroup-5-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5786cf75, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T07:53:50,254 DEBUG [RPCClient-NioEventLoopGroup-5-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [719ac8e87449,36471,-1] 2024-11-26T07:53:50,254 DEBUG [RPCClient-NioEventLoopGroup-5-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-26T07:53:50,254 DEBUG [RPCClient-NioEventLoopGroup-5-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T07:53:50,256 INFO [HMaster-EventLoopGroup-6-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35604, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-26T07:53:50,258 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1d9d15ba, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T07:53:50,258 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-26T07:53:50,260 DEBUG [RPCClient-NioEventLoopGroup-5-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=719ac8e87449,43013,1732607628743, seqNum=-1] 2024-11-26T07:53:50,260 DEBUG [RPCClient-NioEventLoopGroup-5-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-26T07:53:50,263 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-7-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40758, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-26T07:53:50,266 INFO [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=719ac8e87449,36471,1732607628597 2024-11-26T07:53:50,266 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-26T07:53:50,267 DEBUG [RPCClient-NioEventLoopGroup-5-6 {}] client.AsyncConnectionImpl(321): The fetched master address is 719ac8e87449,36471,1732607628597 2024-11-26T07:53:50,268 DEBUG [RPCClient-NioEventLoopGroup-5-6 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@1430e25a 2024-11-26T07:53:50,268 DEBUG [RPCClient-NioEventLoopGroup-5-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-26T07:53:50,270 INFO [HMaster-EventLoopGroup-6-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35616, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-26T07:53:50,271 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36471 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'testBlockEvictionOnGracefulStop1732607630266', {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-26T07:53:50,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36471 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=testBlockEvictionOnGracefulStop1732607630266 2024-11-26T07:53:50,274 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=testBlockEvictionOnGracefulStop1732607630266 execute state=CREATE_TABLE_PRE_OPERATION 2024-11-26T07:53:50,274 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T07:53:50,274 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36471 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "testBlockEvictionOnGracefulStop1732607630266" procId is: 4 2024-11-26T07:53:50,275 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=testBlockEvictionOnGracefulStop1732607630266 execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-26T07:53:50,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36471 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-26T07:53:50,296 INFO [RegionOpenAndInit-testBlockEvictionOnGracefulStop1732607630266-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 21e7f52621ff432de1aac67fc922954b, NAME => 'testBlockEvictionOnGracefulStop1732607630266,,1732607630270.21e7f52621ff432de1aac67fc922954b.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testBlockEvictionOnGracefulStop1732607630266', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/480cd986-aef2-fe93-7892-18b5e497e91c/293d53df-6485-edb9-9b39-cf743ea8b2f2 2024-11-26T07:53:50,315 DEBUG [RegionOpenAndInit-testBlockEvictionOnGracefulStop1732607630266-pool-0 {}] regionserver.HRegion(898): Instantiated testBlockEvictionOnGracefulStop1732607630266,,1732607630270.21e7f52621ff432de1aac67fc922954b.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-26T07:53:50,315 DEBUG [RegionOpenAndInit-testBlockEvictionOnGracefulStop1732607630266-pool-0 {}] regionserver.HRegion(1722): Closing 21e7f52621ff432de1aac67fc922954b, disabling compactions & flushes 2024-11-26T07:53:50,316 INFO [RegionOpenAndInit-testBlockEvictionOnGracefulStop1732607630266-pool-0 {}] regionserver.HRegion(1755): Closing region testBlockEvictionOnGracefulStop1732607630266,,1732607630270.21e7f52621ff432de1aac67fc922954b. 2024-11-26T07:53:50,316 DEBUG [RegionOpenAndInit-testBlockEvictionOnGracefulStop1732607630266-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on testBlockEvictionOnGracefulStop1732607630266,,1732607630270.21e7f52621ff432de1aac67fc922954b. 2024-11-26T07:53:50,316 DEBUG [RegionOpenAndInit-testBlockEvictionOnGracefulStop1732607630266-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on testBlockEvictionOnGracefulStop1732607630266,,1732607630270.21e7f52621ff432de1aac67fc922954b. after waiting 0 ms 2024-11-26T07:53:50,316 DEBUG [RegionOpenAndInit-testBlockEvictionOnGracefulStop1732607630266-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region testBlockEvictionOnGracefulStop1732607630266,,1732607630270.21e7f52621ff432de1aac67fc922954b. 2024-11-26T07:53:50,316 INFO [RegionOpenAndInit-testBlockEvictionOnGracefulStop1732607630266-pool-0 {}] regionserver.HRegion(1973): Closed testBlockEvictionOnGracefulStop1732607630266,,1732607630270.21e7f52621ff432de1aac67fc922954b. 2024-11-26T07:53:50,316 DEBUG [RegionOpenAndInit-testBlockEvictionOnGracefulStop1732607630266-pool-0 {}] regionserver.HRegion(1676): Region close journal for 21e7f52621ff432de1aac67fc922954b: Waiting for close lock at 1732607630315Disabling compacts and flushes for region at 1732607630315Disabling writes for close at 1732607630316 (+1 ms)Writing region close event to WAL at 1732607630316Closed at 1732607630316 2024-11-26T07:53:50,317 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=testBlockEvictionOnGracefulStop1732607630266 execute state=CREATE_TABLE_ADD_TO_META 2024-11-26T07:53:50,318 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testBlockEvictionOnGracefulStop1732607630266,,1732607630270.21e7f52621ff432de1aac67fc922954b.","families":{"info":[{"qualifier":"regioninfo","vlen":78,"tag":[],"timestamp":"1732607630317"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732607630317"}]},"ts":"1732607630317"} 2024-11-26T07:53:50,321 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-26T07:53:50,323 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=testBlockEvictionOnGracefulStop1732607630266 execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-26T07:53:50,323 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testBlockEvictionOnGracefulStop1732607630266","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732607630323"}]},"ts":"1732607630323"} 2024-11-26T07:53:50,327 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=testBlockEvictionOnGracefulStop1732607630266, state=ENABLING in hbase:meta 2024-11-26T07:53:50,327 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(204): Hosts are {719ac8e87449=0} racks are {/default-rack=0} 2024-11-26T07:53:50,328 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-11-26T07:53:50,328 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-11-26T07:53:50,328 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-11-26T07:53:50,328 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-11-26T07:53:50,328 INFO [PEWorker-3 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-11-26T07:53:50,328 INFO [PEWorker-3 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-11-26T07:53:50,328 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-11-26T07:53:50,328 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1732607630266, region=21e7f52621ff432de1aac67fc922954b, ASSIGN}] 2024-11-26T07:53:50,330 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1732607630266, region=21e7f52621ff432de1aac67fc922954b, ASSIGN 2024-11-26T07:53:50,331 INFO [PEWorker-5 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1732607630266, region=21e7f52621ff432de1aac67fc922954b, ASSIGN; state=OFFLINE, location=719ac8e87449,43013,1732607628743; forceNewPlan=false, retain=false 2024-11-26T07:53:50,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36471 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-26T07:53:50,482 INFO [719ac8e87449:36471 {}] balancer.BaseLoadBalancer(388): Reassigned 1 regions. 1 retained the pre-restart assignment. 2024-11-26T07:53:50,483 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=21e7f52621ff432de1aac67fc922954b, regionState=OPENING, regionLocation=719ac8e87449,43013,1732607628743 2024-11-26T07:53:50,486 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-8-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1732607630266, region=21e7f52621ff432de1aac67fc922954b, ASSIGN because future has completed 2024-11-26T07:53:50,487 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 21e7f52621ff432de1aac67fc922954b, server=719ac8e87449,43013,1732607628743}] 2024-11-26T07:53:50,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36471 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-26T07:53:50,611 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-26T07:53:50,643 INFO [RS_OPEN_REGION-regionserver/719ac8e87449:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(132): Open testBlockEvictionOnGracefulStop1732607630266,,1732607630270.21e7f52621ff432de1aac67fc922954b. 2024-11-26T07:53:50,643 DEBUG [RS_OPEN_REGION-regionserver/719ac8e87449:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7752): Opening region: {ENCODED => 21e7f52621ff432de1aac67fc922954b, NAME => 'testBlockEvictionOnGracefulStop1732607630266,,1732607630270.21e7f52621ff432de1aac67fc922954b.', STARTKEY => '', ENDKEY => ''} 2024-11-26T07:53:50,644 DEBUG [RS_OPEN_REGION-regionserver/719ac8e87449:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testBlockEvictionOnGracefulStop1732607630266 21e7f52621ff432de1aac67fc922954b 2024-11-26T07:53:50,644 DEBUG [RS_OPEN_REGION-regionserver/719ac8e87449:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(898): Instantiated testBlockEvictionOnGracefulStop1732607630266,,1732607630270.21e7f52621ff432de1aac67fc922954b.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-26T07:53:50,644 DEBUG [RS_OPEN_REGION-regionserver/719ac8e87449:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7794): checking encryption for 21e7f52621ff432de1aac67fc922954b 2024-11-26T07:53:50,644 DEBUG [RS_OPEN_REGION-regionserver/719ac8e87449:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7797): checking classloading for 21e7f52621ff432de1aac67fc922954b 2024-11-26T07:53:50,645 INFO [StoreOpener-21e7f52621ff432de1aac67fc922954b-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family family of region 21e7f52621ff432de1aac67fc922954b 2024-11-26T07:53:50,649 INFO [StoreOpener-21e7f52621ff432de1aac67fc922954b-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 21e7f52621ff432de1aac67fc922954b columnFamilyName family 2024-11-26T07:53:50,649 DEBUG [StoreOpener-21e7f52621ff432de1aac67fc922954b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T07:53:50,649 INFO [StoreOpener-21e7f52621ff432de1aac67fc922954b-1 {}] regionserver.HStore(327): Store=21e7f52621ff432de1aac67fc922954b/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-26T07:53:50,650 DEBUG [RS_OPEN_REGION-regionserver/719ac8e87449:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1038): replaying wal for 21e7f52621ff432de1aac67fc922954b 2024-11-26T07:53:50,651 DEBUG [RS_OPEN_REGION-regionserver/719ac8e87449:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/480cd986-aef2-fe93-7892-18b5e497e91c/293d53df-6485-edb9-9b39-cf743ea8b2f2/data/default/testBlockEvictionOnGracefulStop1732607630266/21e7f52621ff432de1aac67fc922954b 2024-11-26T07:53:50,651 DEBUG [RS_OPEN_REGION-regionserver/719ac8e87449:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/480cd986-aef2-fe93-7892-18b5e497e91c/293d53df-6485-edb9-9b39-cf743ea8b2f2/data/default/testBlockEvictionOnGracefulStop1732607630266/21e7f52621ff432de1aac67fc922954b 2024-11-26T07:53:50,651 DEBUG [RS_OPEN_REGION-regionserver/719ac8e87449:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1048): stopping wal replay for 21e7f52621ff432de1aac67fc922954b 2024-11-26T07:53:50,651 DEBUG [RS_OPEN_REGION-regionserver/719ac8e87449:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1060): Cleaning up temporary data for 21e7f52621ff432de1aac67fc922954b 2024-11-26T07:53:50,652 DEBUG [RS_OPEN_REGION-regionserver/719ac8e87449:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1093): writing seq id for 21e7f52621ff432de1aac67fc922954b 2024-11-26T07:53:50,667 DEBUG [RS_OPEN_REGION-regionserver/719ac8e87449:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/480cd986-aef2-fe93-7892-18b5e497e91c/293d53df-6485-edb9-9b39-cf743ea8b2f2/data/default/testBlockEvictionOnGracefulStop1732607630266/21e7f52621ff432de1aac67fc922954b/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-26T07:53:50,667 INFO [RS_OPEN_REGION-regionserver/719ac8e87449:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1114): Opened 21e7f52621ff432de1aac67fc922954b; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=62237168, jitterRate=-0.07259392738342285}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-26T07:53:50,667 DEBUG [RS_OPEN_REGION-regionserver/719ac8e87449:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 21e7f52621ff432de1aac67fc922954b 2024-11-26T07:53:50,669 DEBUG [RS_OPEN_REGION-regionserver/719ac8e87449:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1006): Region open journal for 21e7f52621ff432de1aac67fc922954b: Running coprocessor pre-open hook at 1732607630644Writing region info on filesystem at 1732607630644Initializing all the Stores at 1732607630644Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732607630644Cleaning up temporary data from old regions at 1732607630651 (+7 ms)Running coprocessor post-open hooks at 1732607630668 (+17 ms)Region opened successfully at 1732607630669 (+1 ms) 2024-11-26T07:53:50,669 INFO [RS_OPEN_REGION-regionserver/719ac8e87449:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2236): Post open deploy tasks for testBlockEvictionOnGracefulStop1732607630266,,1732607630270.21e7f52621ff432de1aac67fc922954b., pid=6, masterSystemTime=1732607630640 2024-11-26T07:53:50,674 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=21e7f52621ff432de1aac67fc922954b, regionState=OPEN, openSeqNum=2, regionLocation=719ac8e87449,43013,1732607628743 2024-11-26T07:53:50,677 DEBUG [RS_OPEN_REGION-regionserver/719ac8e87449:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2266): Finished post open deploy task for testBlockEvictionOnGracefulStop1732607630266,,1732607630270.21e7f52621ff432de1aac67fc922954b. 2024-11-26T07:53:50,677 INFO [RS_OPEN_REGION-regionserver/719ac8e87449:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(153): Opened testBlockEvictionOnGracefulStop1732607630266,,1732607630270.21e7f52621ff432de1aac67fc922954b. 2024-11-26T07:53:50,678 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-8-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 21e7f52621ff432de1aac67fc922954b, server=719ac8e87449,43013,1732607628743 because future has completed 2024-11-26T07:53:50,684 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=6, resume processing ppid=5 2024-11-26T07:53:50,684 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure 21e7f52621ff432de1aac67fc922954b, server=719ac8e87449,43013,1732607628743 in 194 msec 2024-11-26T07:53:50,687 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=5, resume processing ppid=4 2024-11-26T07:53:50,687 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1732607630266, region=21e7f52621ff432de1aac67fc922954b, ASSIGN in 356 msec 2024-11-26T07:53:50,688 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=testBlockEvictionOnGracefulStop1732607630266 execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-26T07:53:50,688 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testBlockEvictionOnGracefulStop1732607630266","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732607630688"}]},"ts":"1732607630688"} 2024-11-26T07:53:50,691 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=testBlockEvictionOnGracefulStop1732607630266, state=ENABLED in hbase:meta 2024-11-26T07:53:50,691 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=testBlockEvictionOnGracefulStop1732607630266 execute state=CREATE_TABLE_POST_OPERATION 2024-11-26T07:53:50,693 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=testBlockEvictionOnGracefulStop1732607630266 in 421 msec 2024-11-26T07:53:50,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36471 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-26T07:53:50,909 INFO [RPCClient-NioEventLoopGroup-5-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:testBlockEvictionOnGracefulStop1732607630266 completed 2024-11-26T07:53:50,909 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table testBlockEvictionOnGracefulStop1732607630266 get assigned. Timeout = 60000ms 2024-11-26T07:53:50,909 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-26T07:53:50,913 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table testBlockEvictionOnGracefulStop1732607630266 assigned to meta. Checking AM states. 2024-11-26T07:53:50,914 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-26T07:53:50,914 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table testBlockEvictionOnGracefulStop1732607630266 assigned. 2024-11-26T07:53:50,918 DEBUG [RPCClient-NioEventLoopGroup-5-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testBlockEvictionOnGracefulStop1732607630266', row='row1', locateType=CURRENT is [region=testBlockEvictionOnGracefulStop1732607630266,,1732607630270.21e7f52621ff432de1aac67fc922954b., hostname=719ac8e87449,43013,1732607628743, seqNum=2] 2024-11-26T07:53:50,925 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 21e7f52621ff432de1aac67fc922954b 1/1 column families, dataSize=78 B heapSize=480 B 2024-11-26T07:53:50,935 DEBUG [Time-limited test {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 8ca848c697dc4aa393de62dc5bbc9f09 2024-11-26T07:53:50,948 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/480cd986-aef2-fe93-7892-18b5e497e91c/293d53df-6485-edb9-9b39-cf743ea8b2f2/data/default/testBlockEvictionOnGracefulStop1732607630266/21e7f52621ff432de1aac67fc922954b/.tmp/family/8ca848c697dc4aa393de62dc5bbc9f09 is 43, key is row1/family:qf1/1/Put/seqid=0 2024-11-26T07:53:50,949 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=78 B at sequenceid=6 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/480cd986-aef2-fe93-7892-18b5e497e91c/293d53df-6485-edb9-9b39-cf743ea8b2f2/data/default/testBlockEvictionOnGracefulStop1732607630266/21e7f52621ff432de1aac67fc922954b/.tmp/family/8ca848c697dc4aa393de62dc5bbc9f09 2024-11-26T07:53:50,957 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/480cd986-aef2-fe93-7892-18b5e497e91c/293d53df-6485-edb9-9b39-cf743ea8b2f2/data/default/testBlockEvictionOnGracefulStop1732607630266/21e7f52621ff432de1aac67fc922954b/.tmp/family/8ca848c697dc4aa393de62dc5bbc9f09 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/480cd986-aef2-fe93-7892-18b5e497e91c/293d53df-6485-edb9-9b39-cf743ea8b2f2/data/default/testBlockEvictionOnGracefulStop1732607630266/21e7f52621ff432de1aac67fc922954b/family/8ca848c697dc4aa393de62dc5bbc9f09 2024-11-26T07:53:50,958 DEBUG [Time-limited test {}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/480cd986-aef2-fe93-7892-18b5e497e91c/293d53df-6485-edb9-9b39-cf743ea8b2f2/data/default/testBlockEvictionOnGracefulStop1732607630266/21e7f52621ff432de1aac67fc922954b/family/8ca848c697dc4aa393de62dc5bbc9f09, delay=905 ms 2024-11-26T07:53:50,960 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/480cd986-aef2-fe93-7892-18b5e497e91c/293d53df-6485-edb9-9b39-cf743ea8b2f2/data/default/testBlockEvictionOnGracefulStop1732607630266/21e7f52621ff432de1aac67fc922954b/family/8ca848c697dc4aa393de62dc5bbc9f09, entries=2, sequenceid=6, filesize=4.9 K 2024-11-26T07:53:50,962 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~78 B/78, heapSize ~464 B/464, currentSize=0 B/0 for 21e7f52621ff432de1aac67fc922954b in 37ms, sequenceid=6, compaction requested=false 2024-11-26T07:53:50,962 DEBUG [Time-limited test {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'testBlockEvictionOnGracefulStop1732607630266' 2024-11-26T07:53:50,963 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for 21e7f52621ff432de1aac67fc922954b: 2024-11-26T07:53:51,806 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta 2024-11-26T07:53:51,807 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_meta Metrics about Tables on a single HBase RegionServer 2024-11-26T07:53:51,810 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-26T07:53:51,810 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint Metrics about HBase RegionObservers 2024-11-26T07:53:51,811 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_master_table_store 2024-11-26T07:53:51,811 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_master_table_store Metrics about Tables on a single HBase RegionServer 2024-11-26T07:53:51,812 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testBlockEvictionOnGracefulStop1732607630266 2024-11-26T07:53:51,812 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testBlockEvictionOnGracefulStop1732607630266 Metrics about Tables on a single HBase RegionServer 2024-11-26T07:53:51,864 DEBUG [hfile-prefetch-1732607626343 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 8ca848c697dc4aa393de62dc5bbc9f09_0. 2024-11-26T07:53:51,865 DEBUG [hfile-prefetch-1732607626343 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 8ca848c697dc4aa393de62dc5bbc9f09_117. 2024-11-26T07:53:51,865 DEBUG [hfile-prefetch-1732607626343 {}] bucket.BucketCache(2281): Notifying caching completed for file file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/480cd986-aef2-fe93-7892-18b5e497e91c/293d53df-6485-edb9-9b39-cf743ea8b2f2/data/default/testBlockEvictionOnGracefulStop1732607630266/21e7f52621ff432de1aac67fc922954b/family/8ca848c697dc4aa393de62dc5bbc9f09, with total blocks 2, and data blocks 1 2024-11-26T07:53:51,865 DEBUG [hfile-prefetch-1732607626343 {}] bucket.BucketCache(2285): iterating over 0 entries in the backing map 2024-11-26T07:53:51,865 DEBUG [hfile-prefetch-1732607626343 {}] bucket.BucketCache(2308): Prefetch executor completed for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/480cd986-aef2-fe93-7892-18b5e497e91c/293d53df-6485-edb9-9b39-cf743ea8b2f2/data/default/testBlockEvictionOnGracefulStop1732607630266/21e7f52621ff432de1aac67fc922954b/family/8ca848c697dc4aa393de62dc5bbc9f09, but only 0 data blocks were cached. Total data blocks for file: 1. Checking for blocks pending cache in cache writer queue. 2024-11-26T07:53:51,865 DEBUG [hfile-prefetch-1732607626343 {}] bucket.BucketCache(2318): There are still blocks pending caching for file file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/480cd986-aef2-fe93-7892-18b5e497e91c/293d53df-6485-edb9-9b39-cf743ea8b2f2/data/default/testBlockEvictionOnGracefulStop1732607630266/21e7f52621ff432de1aac67fc922954b/family/8ca848c697dc4aa393de62dc5bbc9f09. Will sleep 100ms and try the verification again. 2024-11-26T07:53:51,963 INFO [Time-limited test {}] hbase.SingleProcessHBaseCluster(504): Stopping Thread[RS:0;719ac8e87449:43013,5,FailOnTimeoutGroup] 2024-11-26T07:53:51,964 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '719ac8e87449,43013,1732607628743' ***** 2024-11-26T07:53:51,964 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Stopping rs 0 2024-11-26T07:53:51,964 INFO [RS:0;719ac8e87449:43013 {}] bucket.BucketCache(1738): Disabling cache 2024-11-26T07:53:51,965 DEBUG [hfile-prefetch-1732607626343 {}] bucket.BucketCache(2281): Notifying caching completed for file file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/480cd986-aef2-fe93-7892-18b5e497e91c/293d53df-6485-edb9-9b39-cf743ea8b2f2/data/default/testBlockEvictionOnGracefulStop1732607630266/21e7f52621ff432de1aac67fc922954b/family/8ca848c697dc4aa393de62dc5bbc9f09, with total blocks 2, and data blocks 1 2024-11-26T07:53:51,966 DEBUG [hfile-prefetch-1732607626343 {}] bucket.BucketCache(2285): iterating over 0 entries in the backing map 2024-11-26T07:53:51,966 DEBUG [hfile-prefetch-1732607626343 {}] bucket.BucketCache(2308): Prefetch executor completed for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/480cd986-aef2-fe93-7892-18b5e497e91c/293d53df-6485-edb9-9b39-cf743ea8b2f2/data/default/testBlockEvictionOnGracefulStop1732607630266/21e7f52621ff432de1aac67fc922954b/family/8ca848c697dc4aa393de62dc5bbc9f09, but only 0 data blocks were cached. Total data blocks for file: 1. Checking for blocks pending cache in cache writer queue. 2024-11-26T07:53:51,966 DEBUG [hfile-prefetch-1732607626343 {}] bucket.BucketCache(2318): There are still blocks pending caching for file file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/480cd986-aef2-fe93-7892-18b5e497e91c/293d53df-6485-edb9-9b39-cf743ea8b2f2/data/default/testBlockEvictionOnGracefulStop1732607630266/21e7f52621ff432de1aac67fc922954b/family/8ca848c697dc4aa393de62dc5bbc9f09. Will sleep 100ms and try the verification again. 2024-11-26T07:53:52,020 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-26T07:53:52,037 INFO [regionserver/719ac8e87449:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-26T07:53:52,066 DEBUG [hfile-prefetch-1732607626343 {}] bucket.BucketCache(2281): Notifying caching completed for file file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/480cd986-aef2-fe93-7892-18b5e497e91c/293d53df-6485-edb9-9b39-cf743ea8b2f2/data/default/testBlockEvictionOnGracefulStop1732607630266/21e7f52621ff432de1aac67fc922954b/family/8ca848c697dc4aa393de62dc5bbc9f09, with total blocks 2, and data blocks 1 2024-11-26T07:53:52,066 DEBUG [hfile-prefetch-1732607626343 {}] bucket.BucketCache(2285): iterating over 0 entries in the backing map 2024-11-26T07:53:52,066 DEBUG [hfile-prefetch-1732607626343 {}] bucket.BucketCache(2308): Prefetch executor completed for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/480cd986-aef2-fe93-7892-18b5e497e91c/293d53df-6485-edb9-9b39-cf743ea8b2f2/data/default/testBlockEvictionOnGracefulStop1732607630266/21e7f52621ff432de1aac67fc922954b/family/8ca848c697dc4aa393de62dc5bbc9f09, but only 0 data blocks were cached. Total data blocks for file: 1. Checking for blocks pending cache in cache writer queue. 2024-11-26T07:53:52,066 DEBUG [hfile-prefetch-1732607626343 {}] bucket.BucketCache(2318): There are still blocks pending caching for file file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/480cd986-aef2-fe93-7892-18b5e497e91c/293d53df-6485-edb9-9b39-cf743ea8b2f2/data/default/testBlockEvictionOnGracefulStop1732607630266/21e7f52621ff432de1aac67fc922954b/family/8ca848c697dc4aa393de62dc5bbc9f09. Will sleep 100ms and try the verification again. 2024-11-26T07:53:52,166 DEBUG [hfile-prefetch-1732607626343 {}] bucket.BucketCache(2281): Notifying caching completed for file file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/480cd986-aef2-fe93-7892-18b5e497e91c/293d53df-6485-edb9-9b39-cf743ea8b2f2/data/default/testBlockEvictionOnGracefulStop1732607630266/21e7f52621ff432de1aac67fc922954b/family/8ca848c697dc4aa393de62dc5bbc9f09, with total blocks 2, and data blocks 1 2024-11-26T07:53:52,167 DEBUG [hfile-prefetch-1732607626343 {}] bucket.BucketCache(2285): iterating over 0 entries in the backing map 2024-11-26T07:53:52,167 DEBUG [hfile-prefetch-1732607626343 {}] bucket.BucketCache(2308): Prefetch executor completed for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/480cd986-aef2-fe93-7892-18b5e497e91c/293d53df-6485-edb9-9b39-cf743ea8b2f2/data/default/testBlockEvictionOnGracefulStop1732607630266/21e7f52621ff432de1aac67fc922954b/family/8ca848c697dc4aa393de62dc5bbc9f09, but only 0 data blocks were cached. Total data blocks for file: 1. Checking for blocks pending cache in cache writer queue. 2024-11-26T07:53:52,167 DEBUG [hfile-prefetch-1732607626343 {}] bucket.BucketCache(2318): There are still blocks pending caching for file file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/480cd986-aef2-fe93-7892-18b5e497e91c/293d53df-6485-edb9-9b39-cf743ea8b2f2/data/default/testBlockEvictionOnGracefulStop1732607630266/21e7f52621ff432de1aac67fc922954b/family/8ca848c697dc4aa393de62dc5bbc9f09. Will sleep 100ms and try the verification again. 2024-11-26T07:53:52,171 INFO [RS:0;719ac8e87449:43013 {}] bucket.BucketCache(1762): Shutdown bucket cache: IO persistent=true; path to write=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3acb3e4a-dea5-ea8d-f2d0-bb92c503e71b/bucket.persistence 2024-11-26T07:53:52,171 INFO [Time-limited test-BucketCacheWriter-0 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-0 exiting, cacheEnabled=false 2024-11-26T07:53:52,171 INFO [Time-limited test-BucketCacheWriter-1 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-1 exiting, cacheEnabled=false 2024-11-26T07:53:52,171 INFO [Time-limited test-BucketCacheWriter-2 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-2 exiting, cacheEnabled=false 2024-11-26T07:53:52,171 INFO [RS:0;719ac8e87449:43013 {}] bucket.BucketCache(1768): Shutting down cache persister thread. 2024-11-26T07:53:52,171 WARN [bucket-cache-persister {}] bucket.BucketCachePersister(64): Interrupting BucketCachePersister thread. java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCachePersister.run(BucketCachePersister.java:45) ~[classes/:?] 2024-11-26T07:53:52,181 DEBUG [RS:0;719ac8e87449:43013 {}] bucket.BucketCache(1487): Thread RS:0;719ac8e87449:43013 started persisting bucket cache to file 2024-11-26T07:53:52,181 DEBUG [RS:0;719ac8e87449:43013 {}] bucket.BucketCache(1494): Persist in new chunked persistence format. 2024-11-26T07:53:52,181 DEBUG [RS:0;719ac8e87449:43013 {}] bucket.BucketCache(1673): persistToFile: before persisting backing map size: 1, fullycachedFiles size: 0, chunkSize: 10000 2024-11-26T07:53:52,189 DEBUG [RS:0;719ac8e87449:43013 {}] bucket.BucketCache(1680): persistToFile: after persisting backing map size: 1, fullycachedFiles size: 0 2024-11-26T07:53:52,189 DEBUG [RS:0;719ac8e87449:43013 {}] bucket.BucketCache(1498): PersistToFile: after persisting backing map size: 1, fullycachedFiles size: 0, file name: bucket.persistence1732607632181 2024-11-26T07:53:52,189 DEBUG [RS:0;719ac8e87449:43013 {}] bucket.BucketCache(1509): Thread RS:0;719ac8e87449:43013 finished persisting bucket cache to file, renaming 2024-11-26T07:53:52,190 INFO [RS:0;719ac8e87449:43013 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-26T07:53:52,190 INFO [RS:0;719ac8e87449:43013 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-26T07:53:52,190 INFO [RS:0;719ac8e87449:43013 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-26T07:53:52,190 INFO [RS:0;719ac8e87449:43013 {}] regionserver.HRegionServer(3091): Received CLOSE for 21e7f52621ff432de1aac67fc922954b 2024-11-26T07:53:52,190 INFO [RS:0;719ac8e87449:43013 {}] regionserver.HRegionServer(959): stopping server 719ac8e87449,43013,1732607628743 2024-11-26T07:53:52,190 INFO [RS:0;719ac8e87449:43013 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-26T07:53:52,190 INFO [RS:0;719ac8e87449:43013 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;719ac8e87449:43013. 2024-11-26T07:53:52,190 INFO [RS_CLOSE_REGION-regionserver/719ac8e87449:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1662): Closing region testBlockEvictionOnGracefulStop1732607630266,,1732607630270.21e7f52621ff432de1aac67fc922954b. during a graceful stop, and cache persistence is on, so setting evict on close to false. 2024-11-26T07:53:52,190 DEBUG [RS_CLOSE_REGION-regionserver/719ac8e87449:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 21e7f52621ff432de1aac67fc922954b, disabling compactions & flushes 2024-11-26T07:53:52,190 DEBUG [RS:0;719ac8e87449:43013 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-26T07:53:52,191 INFO [RS_CLOSE_REGION-regionserver/719ac8e87449:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testBlockEvictionOnGracefulStop1732607630266,,1732607630270.21e7f52621ff432de1aac67fc922954b. 2024-11-26T07:53:52,191 DEBUG [RS:0;719ac8e87449:43013 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T07:53:52,191 DEBUG [RS_CLOSE_REGION-regionserver/719ac8e87449:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testBlockEvictionOnGracefulStop1732607630266,,1732607630270.21e7f52621ff432de1aac67fc922954b. 2024-11-26T07:53:52,191 INFO [RS:0;719ac8e87449:43013 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-26T07:53:52,191 DEBUG [RS_CLOSE_REGION-regionserver/719ac8e87449:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testBlockEvictionOnGracefulStop1732607630266,,1732607630270.21e7f52621ff432de1aac67fc922954b. after waiting 0 ms 2024-11-26T07:53:52,191 INFO [RS:0;719ac8e87449:43013 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-26T07:53:52,191 DEBUG [RS_CLOSE_REGION-regionserver/719ac8e87449:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testBlockEvictionOnGracefulStop1732607630266,,1732607630270.21e7f52621ff432de1aac67fc922954b. 2024-11-26T07:53:52,191 INFO [RS:0;719ac8e87449:43013 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-26T07:53:52,191 INFO [RS:0;719ac8e87449:43013 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-11-26T07:53:52,191 INFO [RS:0;719ac8e87449:43013 {}] regionserver.HRegionServer(1321): Waiting on 2 regions to close 2024-11-26T07:53:52,191 DEBUG [RS:0;719ac8e87449:43013 {}] regionserver.HRegionServer(1325): Online Regions={21e7f52621ff432de1aac67fc922954b=testBlockEvictionOnGracefulStop1732607630266,,1732607630270.21e7f52621ff432de1aac67fc922954b., 1588230740=hbase:meta,,1.1588230740} 2024-11-26T07:53:52,191 DEBUG [RS:0;719ac8e87449:43013 {}] regionserver.HRegionServer(1351): Waiting on 1588230740, 21e7f52621ff432de1aac67fc922954b 2024-11-26T07:53:52,191 INFO [RS_CLOSE_META-regionserver/719ac8e87449:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1662): Closing region hbase:meta,,1.1588230740 during a graceful stop, and cache persistence is on, so setting evict on close to false. 2024-11-26T07:53:52,191 DEBUG [RS_CLOSE_META-regionserver/719ac8e87449:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-26T07:53:52,191 DEBUG [StoreFileCloser-21e7f52621ff432de1aac67fc922954b-family-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/480cd986-aef2-fe93-7892-18b5e497e91c/293d53df-6485-edb9-9b39-cf743ea8b2f2/data/default/testBlockEvictionOnGracefulStop1732607630266/21e7f52621ff432de1aac67fc922954b/family/8ca848c697dc4aa393de62dc5bbc9f09 2024-11-26T07:53:52,191 INFO [RS_CLOSE_META-regionserver/719ac8e87449:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-26T07:53:52,191 DEBUG [RS_CLOSE_META-regionserver/719ac8e87449:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-26T07:53:52,191 DEBUG [RS_CLOSE_META-regionserver/719ac8e87449:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-26T07:53:52,192 DEBUG [RS_CLOSE_META-regionserver/719ac8e87449:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-26T07:53:52,192 INFO [RS_CLOSE_META-regionserver/719ac8e87449:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=1.75 KB heapSize=3.78 KB 2024-11-26T07:53:52,194 WARN [hfile-prefetch-1732607626343 {}] hfile.HFilePreadReader$1(158): Prefetch path=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/480cd986-aef2-fe93-7892-18b5e497e91c/293d53df-6485-edb9-9b39-cf743ea8b2f2/data/default/testBlockEvictionOnGracefulStop1732607630266/21e7f52621ff432de1aac67fc922954b/family/8ca848c697dc4aa393de62dc5bbc9f09, offset=158, end=158 java.lang.RuntimeException: java.lang.InterruptedException: sleep interrupted at org.apache.hadoop.hbase.io.hfile.bucket.BucketCache.notifyFileCachingCompleted(BucketCache.java:2337) ~[classes/:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCache.notifyFileCachingCompleted(BucketCache.java:2321) ~[classes/:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCache.notifyFileCachingCompleted(BucketCache.java:2321) ~[classes/:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCache.notifyFileCachingCompleted(BucketCache.java:2321) ~[classes/:?] at org.apache.hadoop.hbase.io.hfile.CombinedBlockCache.notifyFileCachingCompleted(CombinedBlockCache.java:469) ~[classes/:?] at org.apache.hadoop.hbase.io.hfile.HFilePreadReader$1.run(HFilePreadReader.java:148) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.lambda$tracedRunnable$2(TraceUtil.java:155) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:539) ~[?:?] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:304) ~[?:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCache.notifyFileCachingCompleted(BucketCache.java:2320) ~[classes/:?] ... 12 more 2024-11-26T07:53:52,194 DEBUG [hfile-prefetch-1732607626343 {}] hfile.PrefetchExecutor(121): Prefetch completed for 8ca848c697dc4aa393de62dc5bbc9f09 2024-11-26T07:53:52,200 DEBUG [RS_CLOSE_REGION-regionserver/719ac8e87449:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/480cd986-aef2-fe93-7892-18b5e497e91c/293d53df-6485-edb9-9b39-cf743ea8b2f2/data/default/testBlockEvictionOnGracefulStop1732607630266/21e7f52621ff432de1aac67fc922954b/recovered.edits/9.seqid, newMaxSeqId=9, maxSeqId=1 2024-11-26T07:53:52,200 INFO [RS_CLOSE_REGION-regionserver/719ac8e87449:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testBlockEvictionOnGracefulStop1732607630266,,1732607630270.21e7f52621ff432de1aac67fc922954b. 2024-11-26T07:53:52,201 DEBUG [RS_CLOSE_REGION-regionserver/719ac8e87449:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 21e7f52621ff432de1aac67fc922954b: Waiting for close lock at 1732607632190Running coprocessor pre-close hooks at 1732607632190Disabling compacts and flushes for region at 1732607632190Disabling writes for close at 1732607632191 (+1 ms)Writing region close event to WAL at 1732607632192 (+1 ms)Running coprocessor post-close hooks at 1732607632200 (+8 ms)Closed at 1732607632200 2024-11-26T07:53:52,201 DEBUG [RS_CLOSE_REGION-regionserver/719ac8e87449:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testBlockEvictionOnGracefulStop1732607630266,,1732607630270.21e7f52621ff432de1aac67fc922954b. 2024-11-26T07:53:52,212 DEBUG [RS_CLOSE_META-regionserver/719ac8e87449:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 10e8dbaa8e7e42af8dad71e08a48806d 2024-11-26T07:53:52,226 DEBUG [RS_CLOSE_META-regionserver/719ac8e87449:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/480cd986-aef2-fe93-7892-18b5e497e91c/293d53df-6485-edb9-9b39-cf743ea8b2f2/data/hbase/meta/1588230740/.tmp/info/10e8dbaa8e7e42af8dad71e08a48806d is 209, key is testBlockEvictionOnGracefulStop1732607630266,,1732607630270.21e7f52621ff432de1aac67fc922954b./info:regioninfo/1732607630674/Put/seqid=0 2024-11-26T07:53:52,227 INFO [RS_CLOSE_META-regionserver/719ac8e87449:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.53 KB at sequenceid=11 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/480cd986-aef2-fe93-7892-18b5e497e91c/293d53df-6485-edb9-9b39-cf743ea8b2f2/data/hbase/meta/1588230740/.tmp/info/10e8dbaa8e7e42af8dad71e08a48806d 2024-11-26T07:53:52,230 DEBUG [bucket-cache-persister {}] bucket.BucketCachePersister(47): Cache is inconsistent, persisting to disk 2024-11-26T07:53:52,230 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1487): Thread bucket-cache-persister started persisting bucket cache to file 2024-11-26T07:53:52,230 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1494): Persist in new chunked persistence format. 2024-11-26T07:53:52,230 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1673): persistToFile: before persisting backing map size: 0, fullycachedFiles size: 0, chunkSize: 10000 2024-11-26T07:53:52,235 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1680): persistToFile: after persisting backing map size: 0, fullycachedFiles size: 0 2024-11-26T07:53:52,235 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1498): PersistToFile: after persisting backing map size: 0, fullycachedFiles size: 0, file name: bucket.persistence1732607632230 2024-11-26T07:53:52,235 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1509): Thread bucket-cache-persister finished persisting bucket cache to file, renaming 2024-11-26T07:53:52,247 DEBUG [RS_CLOSE_META-regionserver/719ac8e87449:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFile$WriterFactory(308): Unable to set drop behind on be72219bda5746729a3db47780310244 2024-11-26T07:53:52,261 DEBUG [RS_CLOSE_META-regionserver/719ac8e87449:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/480cd986-aef2-fe93-7892-18b5e497e91c/293d53df-6485-edb9-9b39-cf743ea8b2f2/data/hbase/meta/1588230740/.tmp/ns/be72219bda5746729a3db47780310244 is 43, key is default/ns:d/1732607630129/Put/seqid=0 2024-11-26T07:53:52,261 INFO [RS_CLOSE_META-regionserver/719ac8e87449:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=74 B at sequenceid=11 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/480cd986-aef2-fe93-7892-18b5e497e91c/293d53df-6485-edb9-9b39-cf743ea8b2f2/data/hbase/meta/1588230740/.tmp/ns/be72219bda5746729a3db47780310244 2024-11-26T07:53:52,271 DEBUG [RS_CLOSE_META-regionserver/719ac8e87449:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 46047b57f83540a88309c73fe3773775 2024-11-26T07:53:52,289 DEBUG [RS_CLOSE_META-regionserver/719ac8e87449:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/480cd986-aef2-fe93-7892-18b5e497e91c/293d53df-6485-edb9-9b39-cf743ea8b2f2/data/hbase/meta/1588230740/.tmp/table/46047b57f83540a88309c73fe3773775 is 80, key is testBlockEvictionOnGracefulStop1732607630266/table:state/1732607630688/Put/seqid=0 2024-11-26T07:53:52,290 INFO [RS_CLOSE_META-regionserver/719ac8e87449:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=152 B at sequenceid=11 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/480cd986-aef2-fe93-7892-18b5e497e91c/293d53df-6485-edb9-9b39-cf743ea8b2f2/data/hbase/meta/1588230740/.tmp/table/46047b57f83540a88309c73fe3773775 2024-11-26T07:53:52,293 DEBUG [RS_CLOSE_META-regionserver/719ac8e87449:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/480cd986-aef2-fe93-7892-18b5e497e91c/293d53df-6485-edb9-9b39-cf743ea8b2f2/data/hbase/meta/1588230740/.tmp/info/10e8dbaa8e7e42af8dad71e08a48806d as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/480cd986-aef2-fe93-7892-18b5e497e91c/293d53df-6485-edb9-9b39-cf743ea8b2f2/data/hbase/meta/1588230740/info/10e8dbaa8e7e42af8dad71e08a48806d 2024-11-26T07:53:52,295 DEBUG [RS_CLOSE_META-regionserver/719ac8e87449:0-0 {event_type=M_RS_CLOSE_META}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/480cd986-aef2-fe93-7892-18b5e497e91c/293d53df-6485-edb9-9b39-cf743ea8b2f2/data/hbase/meta/1588230740/info/10e8dbaa8e7e42af8dad71e08a48806d, delay=956 ms 2024-11-26T07:53:52,296 INFO [RS_CLOSE_META-regionserver/719ac8e87449:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/480cd986-aef2-fe93-7892-18b5e497e91c/293d53df-6485-edb9-9b39-cf743ea8b2f2/data/hbase/meta/1588230740/info/10e8dbaa8e7e42af8dad71e08a48806d, entries=10, sequenceid=11, filesize=7.0 K 2024-11-26T07:53:52,297 DEBUG [RS_CLOSE_META-regionserver/719ac8e87449:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/480cd986-aef2-fe93-7892-18b5e497e91c/293d53df-6485-edb9-9b39-cf743ea8b2f2/data/hbase/meta/1588230740/.tmp/ns/be72219bda5746729a3db47780310244 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/480cd986-aef2-fe93-7892-18b5e497e91c/293d53df-6485-edb9-9b39-cf743ea8b2f2/data/hbase/meta/1588230740/ns/be72219bda5746729a3db47780310244 2024-11-26T07:53:52,298 DEBUG [RS_CLOSE_META-regionserver/719ac8e87449:0-0 {event_type=M_RS_CLOSE_META}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/480cd986-aef2-fe93-7892-18b5e497e91c/293d53df-6485-edb9-9b39-cf743ea8b2f2/data/hbase/meta/1588230740/ns/be72219bda5746729a3db47780310244, delay=911 ms 2024-11-26T07:53:52,300 INFO [RS_CLOSE_META-regionserver/719ac8e87449:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/480cd986-aef2-fe93-7892-18b5e497e91c/293d53df-6485-edb9-9b39-cf743ea8b2f2/data/hbase/meta/1588230740/ns/be72219bda5746729a3db47780310244, entries=2, sequenceid=11, filesize=5.0 K 2024-11-26T07:53:52,300 DEBUG [RS_CLOSE_META-regionserver/719ac8e87449:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/480cd986-aef2-fe93-7892-18b5e497e91c/293d53df-6485-edb9-9b39-cf743ea8b2f2/data/hbase/meta/1588230740/.tmp/table/46047b57f83540a88309c73fe3773775 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/480cd986-aef2-fe93-7892-18b5e497e91c/293d53df-6485-edb9-9b39-cf743ea8b2f2/data/hbase/meta/1588230740/table/46047b57f83540a88309c73fe3773775 2024-11-26T07:53:52,302 DEBUG [RS_CLOSE_META-regionserver/719ac8e87449:0-0 {event_type=M_RS_CLOSE_META}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/480cd986-aef2-fe93-7892-18b5e497e91c/293d53df-6485-edb9-9b39-cf743ea8b2f2/data/hbase/meta/1588230740/table/46047b57f83540a88309c73fe3773775, delay=920 ms 2024-11-26T07:53:52,303 INFO [RS_CLOSE_META-regionserver/719ac8e87449:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/480cd986-aef2-fe93-7892-18b5e497e91c/293d53df-6485-edb9-9b39-cf743ea8b2f2/data/hbase/meta/1588230740/table/46047b57f83540a88309c73fe3773775, entries=2, sequenceid=11, filesize=5.3 K 2024-11-26T07:53:52,306 INFO [RS_CLOSE_META-regionserver/719ac8e87449:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~1.75 KB/1797, heapSize ~3.48 KB/3568, currentSize=0 B/0 for 1588230740 in 114ms, sequenceid=11, compaction requested=false 2024-11-26T07:53:52,307 DEBUG [StoreFileCloser-1588230740-info-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/480cd986-aef2-fe93-7892-18b5e497e91c/293d53df-6485-edb9-9b39-cf743ea8b2f2/data/hbase/meta/1588230740/info/10e8dbaa8e7e42af8dad71e08a48806d 2024-11-26T07:53:52,308 DEBUG [StoreFileCloser-1588230740-ns-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/480cd986-aef2-fe93-7892-18b5e497e91c/293d53df-6485-edb9-9b39-cf743ea8b2f2/data/hbase/meta/1588230740/ns/be72219bda5746729a3db47780310244 2024-11-26T07:53:52,309 DEBUG [StoreFileCloser-1588230740-table-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/480cd986-aef2-fe93-7892-18b5e497e91c/293d53df-6485-edb9-9b39-cf743ea8b2f2/data/hbase/meta/1588230740/table/46047b57f83540a88309c73fe3773775 2024-11-26T07:53:52,319 DEBUG [RS_CLOSE_META-regionserver/719ac8e87449:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/480cd986-aef2-fe93-7892-18b5e497e91c/293d53df-6485-edb9-9b39-cf743ea8b2f2/data/hbase/meta/1588230740/recovered.edits/14.seqid, newMaxSeqId=14, maxSeqId=1 2024-11-26T07:53:52,319 DEBUG [RS_CLOSE_META-regionserver/719ac8e87449:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-26T07:53:52,320 INFO [RS_CLOSE_META-regionserver/719ac8e87449:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-26T07:53:52,320 DEBUG [RS_CLOSE_META-regionserver/719ac8e87449:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1732607632191Running coprocessor pre-close hooks at 1732607632191Disabling compacts and flushes for region at 1732607632191Disabling writes for close at 1732607632192 (+1 ms)Obtaining lock to block concurrent updates at 1732607632192Preparing flush snapshotting stores in 1588230740 at 1732607632192Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=1797, getHeapSize=3808, getOffHeapSize=0, getCellsCount=14 at 1732607632192Flushing stores of hbase:meta,,1.1588230740 at 1732607632194 (+2 ms)Flushing 1588230740/info: creating writer at 1732607632194Flushing 1588230740/info: appending metadata at 1732607632225 (+31 ms)Flushing 1588230740/info: closing flushed file at 1732607632225Flushing 1588230740/ns: creating writer at 1732607632230 (+5 ms)Flushing 1588230740/ns: appending metadata at 1732607632260 (+30 ms)Flushing 1588230740/ns: closing flushed file at 1732607632260Flushing 1588230740/table: creating writer at 1732607632264 (+4 ms)Flushing 1588230740/table: appending metadata at 1732607632288 (+24 ms)Flushing 1588230740/table: closing flushed file at 1732607632288Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@22311ed2: reopening flushed file at 1732607632293 (+5 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5cf73381: reopening flushed file at 1732607632297 (+4 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@278ec8e6: reopening flushed file at 1732607632300 (+3 ms)Finished flush of dataSize ~1.75 KB/1797, heapSize ~3.48 KB/3568, currentSize=0 B/0 for 1588230740 in 114ms, sequenceid=11, compaction requested=false at 1732607632306 (+6 ms)Writing region close event to WAL at 1732607632310 (+4 ms)Running coprocessor post-close hooks at 1732607632319 (+9 ms)Closed at 1732607632320 (+1 ms) 2024-11-26T07:53:52,320 DEBUG [RS_CLOSE_META-regionserver/719ac8e87449:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-11-26T07:53:52,391 INFO [RS:0;719ac8e87449:43013 {}] regionserver.HRegionServer(976): stopping server 719ac8e87449,43013,1732607628743; all regions closed. 2024-11-26T07:53:52,394 DEBUG [RS:0;719ac8e87449:43013 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/480cd986-aef2-fe93-7892-18b5e497e91c/293d53df-6485-edb9-9b39-cf743ea8b2f2/oldWALs 2024-11-26T07:53:52,394 INFO [RS:0;719ac8e87449:43013 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 719ac8e87449%2C43013%2C1732607628743.meta:.meta(num 1732607630042) 2024-11-26T07:53:52,396 DEBUG [RS:0;719ac8e87449:43013 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/480cd986-aef2-fe93-7892-18b5e497e91c/293d53df-6485-edb9-9b39-cf743ea8b2f2/oldWALs 2024-11-26T07:53:52,396 INFO [RS:0;719ac8e87449:43013 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 719ac8e87449%2C43013%2C1732607628743:(num 1732607629670) 2024-11-26T07:53:52,396 DEBUG [RS:0;719ac8e87449:43013 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T07:53:52,396 INFO [RS:0;719ac8e87449:43013 {}] regionserver.LeaseManager(133): Closed leases 2024-11-26T07:53:52,396 INFO [RS:0;719ac8e87449:43013 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-26T07:53:52,396 INFO [RS:0;719ac8e87449:43013 {}] hbase.ChoreService(370): Chore service for: regionserver/719ac8e87449:0 had [ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-11-26T07:53:52,396 INFO [RS:0;719ac8e87449:43013 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-26T07:53:52,397 INFO [regionserver/719ac8e87449:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-26T07:53:52,397 INFO [RS:0;719ac8e87449:43013 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:43013 2024-11-26T07:53:52,415 INFO [RS:0;719ac8e87449:43013 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-26T07:53:52,415 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36471-0x101776be7e90000, quorum=127.0.0.1:64716, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-26T07:53:52,415 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43013-0x101776be7e90001, quorum=127.0.0.1:64716, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/719ac8e87449,43013,1732607628743 2024-11-26T07:53:52,424 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [719ac8e87449,43013,1732607628743] 2024-11-26T07:53:52,432 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/719ac8e87449,43013,1732607628743 already deleted, retry=false 2024-11-26T07:53:52,432 INFO [RegionServerTracker-0 {}] master.ServerManager(695): Processing expiration of 719ac8e87449,43013,1732607628743 on 719ac8e87449,36471,1732607628597 2024-11-26T07:53:52,436 DEBUG [RegionServerTracker-0 {}] procedure2.ProcedureExecutor(1139): Stored pid=7, state=RUNNABLE:SERVER_CRASH_START, hasLock=false; ServerCrashProcedure 719ac8e87449,43013,1732607628743, splitWal=true, meta=true 2024-11-26T07:53:52,438 INFO [RegionServerTracker-0 {}] assignment.AssignmentManager(1999): Scheduled ServerCrashProcedure pid=7 for 719ac8e87449,43013,1732607628743 (carryingMeta=true) 719ac8e87449,43013,1732607628743/CRASHED/regionCount=2/lock=java.util.concurrent.locks.ReentrantReadWriteLock@225ea72a[Write locks = 1, Read locks = 0], oldState=ONLINE. 2024-11-26T07:53:52,438 INFO [PEWorker-2 {}] procedure.ServerCrashProcedure(169): Start pid=7, state=RUNNABLE:SERVER_CRASH_START, hasLock=true; ServerCrashProcedure 719ac8e87449,43013,1732607628743, splitWal=true, meta=true 2024-11-26T07:53:52,440 INFO [PEWorker-2 {}] procedure.ServerCrashProcedure(339): Splitting WALs pid=7, state=RUNNABLE:SERVER_CRASH_SPLIT_META_LOGS, hasLock=true; ServerCrashProcedure 719ac8e87449,43013,1732607628743, splitWal=true, meta=true, isMeta: true 2024-11-26T07:53:52,440 DEBUG [PEWorker-2 {}] master.MasterWalManager(329): Renamed region directory: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/480cd986-aef2-fe93-7892-18b5e497e91c/293d53df-6485-edb9-9b39-cf743ea8b2f2/WALs/719ac8e87449,43013,1732607628743-splitting 2024-11-26T07:53:52,440 INFO [PEWorker-2 {}] master.SplitLogManager(171): file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/480cd986-aef2-fe93-7892-18b5e497e91c/293d53df-6485-edb9-9b39-cf743ea8b2f2/WALs/719ac8e87449,43013,1732607628743-splitting dir is empty, no logs to split. 2024-11-26T07:53:52,441 INFO [PEWorker-2 {}] master.SplitWALManager(105): 719ac8e87449,43013,1732607628743 WAL count=0, meta=true 2024-11-26T07:53:52,442 INFO [PEWorker-2 {}] master.SplitLogManager(171): file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/480cd986-aef2-fe93-7892-18b5e497e91c/293d53df-6485-edb9-9b39-cf743ea8b2f2/WALs/719ac8e87449,43013,1732607628743-splitting dir is empty, no logs to split. 2024-11-26T07:53:52,442 INFO [PEWorker-2 {}] master.SplitWALManager(105): 719ac8e87449,43013,1732607628743 WAL count=0, meta=true 2024-11-26T07:53:52,443 DEBUG [PEWorker-2 {}] procedure.ServerCrashProcedure(329): Check if 719ac8e87449,43013,1732607628743 WAL splitting is done? wals=0, meta=true 2024-11-26T07:53:52,444 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=8, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-11-26T07:53:52,445 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=8, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-26T07:53:52,446 INFO [PEWorker-5 {}] assignment.TransitRegionStateProcedure(269): Starting pid=8, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-11-26T07:53:52,487 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/719ac8e87449:0 server-side Connection retries=45 2024-11-26T07:53:52,487 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-26T07:53:52,487 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-26T07:53:52,487 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-26T07:53:52,487 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-26T07:53:52,487 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-26T07:53:52,488 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-26T07:53:52,488 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-26T07:53:52,488 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:38357 2024-11-26T07:53:52,491 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:38357 connecting to ZooKeeper ensemble=127.0.0.1:64716 2024-11-26T07:53:52,491 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-26T07:53:52,491 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-26T07:53:52,521 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:383570x0, quorum=127.0.0.1:64716, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-26T07:53:52,522 DEBUG [Time-limited test {}] zookeeper.ZKUtil(111): regionserver:383570x0, quorum=127.0.0.1:64716, baseZNode=/hbase Set watcher on existing znode=/hbase/running 2024-11-26T07:53:52,522 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:38357-0x101776be7e90003 connected 2024-11-26T07:53:52,522 DEBUG [pool-298-thread-1 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: INIT 2024-11-26T07:53:52,523 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-26T07:53:52,523 INFO [Time-limited test {}] bucket.FileIOEngine(99): Allocating cache 400 MB, on the path:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3acb3e4a-dea5-ea8d-f2d0-bb92c503e71b/bucket.cache 2024-11-26T07:53:52,524 INFO [Time-limited test {}] bucket.BucketCache(364): Instantiating BucketCache with acceptableFactor: 0.95, minFactor: 0.85, extraFreeFactor: 0.1, singleFactor: 0.25, multiFactor: 0.5, memoryFactor: 0.25, useStrongRef: false 2024-11-26T07:53:52,524 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43013-0x101776be7e90001, quorum=127.0.0.1:64716, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-26T07:53:52,524 INFO [RS:0;719ac8e87449:43013 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-26T07:53:52,524 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43013-0x101776be7e90001, quorum=127.0.0.1:64716, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-26T07:53:52,524 INFO [RS:0;719ac8e87449:43013 {}] regionserver.HRegionServer(1031): Exiting; stopping=719ac8e87449,43013,1732607628743; zookeeper connection closed. 2024-11-26T07:53:52,524 INFO [Time-limited test {}] bucket.BucketCache(478): Starting BucketCachePersisterThread 2024-11-26T07:53:52,524 INFO [Time-limited test {}] bucket.BucketCachePersister(38): BucketCachePersister started with interval: 100 2024-11-26T07:53:52,524 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@10050b13 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@10050b13 2024-11-26T07:53:52,524 INFO [Thread-206 {}] bucket.BucketCache(1530): Started retrieving bucket cache from file 2024-11-26T07:53:52,524 INFO [Time-limited test {}] bucket.BucketCache(403): Started bucket cache; ioengine=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3acb3e4a-dea5-ea8d-f2d0-bb92c503e71b/bucket.cache, capacity=400 MB, blockSize=64 KB, writerThreadNum=3, writerQLen=64, persistencePath=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3acb3e4a-dea5-ea8d-f2d0-bb92c503e71b/bucket.persistence, bucketAllocator=org.apache.hadoop.hbase.io.hfile.bucket.BucketAllocator 2024-11-26T07:53:52,525 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-26T07:53:52,525 INFO [Thread-206 {}] bucket.BucketCache(1552): Reading new chunked format of persistence. 2024-11-26T07:53:52,525 DEBUG [Time-limited test {}] zookeeper.ZKUtil(111): regionserver:38357-0x101776be7e90003, quorum=127.0.0.1:64716, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-11-26T07:53:52,526 INFO [Thread-206 {}] bucket.BucketCache(1705): Retrieved 0 of chunks with blockCount = 0. 2024-11-26T07:53:52,527 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:38357-0x101776be7e90003, quorum=127.0.0.1:64716, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-26T07:53:52,535 DEBUG [Thread-206 {}] bucket.BucketCache(1569): Updating region size map after retrieving cached file list 2024-11-26T07:53:52,535 INFO [Thread-206 {}] bucket.BucketAllocator(343): Cache totalSize=418148352, buckets=199, bucket capacity=2101248=(4*525312)=(FEWEST_ITEMS_IN_BUCKET*(largest configured bucketcache size)) 2024-11-26T07:53:52,535 INFO [Thread-206 {}] bucket.BucketCache(1562): Bucket cache retrieved from file successfully with size: 0 2024-11-26T07:53:52,535 INFO [Thread-206 {}] bucket.BucketCache(413): Persistent bucket cache recovery from /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3acb3e4a-dea5-ea8d-f2d0-bb92c503e71b/bucket.persistence is complete. 2024-11-26T07:53:52,538 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=38357 2024-11-26T07:53:52,538 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=38357 2024-11-26T07:53:52,538 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=38357 2024-11-26T07:53:52,542 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=38357 2024-11-26T07:53:52,542 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=38357 2024-11-26T07:53:52,544 INFO [RS:2;719ac8e87449:38357 {}] regionserver.HRegionServer(746): ClusterId : 0bd26192-d019-4cda-81e6-944fe4f54fbe 2024-11-26T07:53:52,544 DEBUG [RS:2;719ac8e87449:38357 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-26T07:53:52,558 DEBUG [RS:2;719ac8e87449:38357 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-26T07:53:52,558 DEBUG [RS:2;719ac8e87449:38357 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-26T07:53:52,566 DEBUG [RS:2;719ac8e87449:38357 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-26T07:53:52,567 DEBUG [RS:2;719ac8e87449:38357 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@8bd4bf8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=719ac8e87449/172.17.0.2:0 2024-11-26T07:53:52,583 DEBUG [RS:2;719ac8e87449:38357 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:2;719ac8e87449:38357 2024-11-26T07:53:52,583 INFO [RS:2;719ac8e87449:38357 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-26T07:53:52,583 INFO [RS:2;719ac8e87449:38357 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-26T07:53:52,583 DEBUG [RS:2;719ac8e87449:38357 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-26T07:53:52,584 INFO [RS:2;719ac8e87449:38357 {}] regionserver.HRegionServer(2659): reportForDuty to master=719ac8e87449,36471,1732607628597 with port=38357, startcode=1732607632486 2024-11-26T07:53:52,584 DEBUG [RS:2;719ac8e87449:38357 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-26T07:53:52,586 INFO [HMaster-EventLoopGroup-6-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56147, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=RegionServerStatusService 2024-11-26T07:53:52,587 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=36471 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 719ac8e87449,38357,1732607632486 2024-11-26T07:53:52,587 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=36471 {}] master.ServerManager(517): Registering regionserver=719ac8e87449,38357,1732607632486 2024-11-26T07:53:52,590 DEBUG [RS:2;719ac8e87449:38357 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/480cd986-aef2-fe93-7892-18b5e497e91c/293d53df-6485-edb9-9b39-cf743ea8b2f2 2024-11-26T07:53:52,590 DEBUG [RS:2;719ac8e87449:38357 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=file:/ 2024-11-26T07:53:52,590 DEBUG [RS:2;719ac8e87449:38357 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-26T07:53:52,596 DEBUG [719ac8e87449:36471 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=1, allServersCount=1 2024-11-26T07:53:52,597 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=8 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=719ac8e87449,34097,1732607628803 2024-11-26T07:53:52,598 INFO [PEWorker-4 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 719ac8e87449,34097,1732607628803, state=OPENING 2024-11-26T07:53:52,599 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36471-0x101776be7e90000, quorum=127.0.0.1:64716, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-26T07:53:52,607 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34097-0x101776be7e90002, quorum=127.0.0.1:64716, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-26T07:53:52,607 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38357-0x101776be7e90003, quorum=127.0.0.1:64716, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-26T07:53:52,607 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36471-0x101776be7e90000, quorum=127.0.0.1:64716, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-26T07:53:52,607 DEBUG [RS:2;719ac8e87449:38357 {}] zookeeper.ZKUtil(111): regionserver:38357-0x101776be7e90003, quorum=127.0.0.1:64716, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/719ac8e87449,38357,1732607632486 2024-11-26T07:53:52,607 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-26T07:53:52,607 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-26T07:53:52,607 WARN [RS:2;719ac8e87449:38357 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-26T07:53:52,607 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-26T07:53:52,607 INFO [RS:2;719ac8e87449:38357 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-26T07:53:52,607 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [719ac8e87449,38357,1732607632486] 2024-11-26T07:53:52,607 DEBUG [RS:2;719ac8e87449:38357 {}] regionserver.HRegionServer(1793): logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/480cd986-aef2-fe93-7892-18b5e497e91c/293d53df-6485-edb9-9b39-cf743ea8b2f2/WALs/719ac8e87449,38357,1732607632486 2024-11-26T07:53:52,608 DEBUG [PEWorker-4 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=8, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-26T07:53:52,608 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=9, ppid=8, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=719ac8e87449,34097,1732607628803}] 2024-11-26T07:53:52,614 INFO [RS:2;719ac8e87449:38357 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-26T07:53:52,616 INFO [RS:2;719ac8e87449:38357 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-26T07:53:52,616 INFO [RS:2;719ac8e87449:38357 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-26T07:53:52,616 INFO [RS:2;719ac8e87449:38357 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-26T07:53:52,617 INFO [RS:2;719ac8e87449:38357 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-26T07:53:52,617 INFO [RS:2;719ac8e87449:38357 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-26T07:53:52,617 INFO [RS:2;719ac8e87449:38357 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-26T07:53:52,617 DEBUG [RS:2;719ac8e87449:38357 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/719ac8e87449:0, corePoolSize=1, maxPoolSize=1 2024-11-26T07:53:52,617 DEBUG [RS:2;719ac8e87449:38357 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/719ac8e87449:0, corePoolSize=1, maxPoolSize=1 2024-11-26T07:53:52,617 DEBUG [RS:2;719ac8e87449:38357 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/719ac8e87449:0, corePoolSize=1, maxPoolSize=1 2024-11-26T07:53:52,618 DEBUG [RS:2;719ac8e87449:38357 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/719ac8e87449:0, corePoolSize=1, maxPoolSize=1 2024-11-26T07:53:52,618 DEBUG [RS:2;719ac8e87449:38357 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/719ac8e87449:0, corePoolSize=1, maxPoolSize=1 2024-11-26T07:53:52,618 DEBUG [RS:2;719ac8e87449:38357 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/719ac8e87449:0, corePoolSize=2, maxPoolSize=2 2024-11-26T07:53:52,618 DEBUG [RS:2;719ac8e87449:38357 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/719ac8e87449:0, corePoolSize=1, maxPoolSize=1 2024-11-26T07:53:52,618 DEBUG [RS:2;719ac8e87449:38357 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/719ac8e87449:0, corePoolSize=1, maxPoolSize=1 2024-11-26T07:53:52,618 DEBUG [RS:2;719ac8e87449:38357 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/719ac8e87449:0, corePoolSize=1, maxPoolSize=1 2024-11-26T07:53:52,618 DEBUG [RS:2;719ac8e87449:38357 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/719ac8e87449:0, corePoolSize=1, maxPoolSize=1 2024-11-26T07:53:52,618 DEBUG [RS:2;719ac8e87449:38357 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/719ac8e87449:0, corePoolSize=1, maxPoolSize=1 2024-11-26T07:53:52,618 DEBUG [RS:2;719ac8e87449:38357 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/719ac8e87449:0, corePoolSize=1, maxPoolSize=1 2024-11-26T07:53:52,618 DEBUG [RS:2;719ac8e87449:38357 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/719ac8e87449:0, corePoolSize=3, maxPoolSize=3 2024-11-26T07:53:52,618 DEBUG [RS:2;719ac8e87449:38357 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/719ac8e87449:0, corePoolSize=3, maxPoolSize=3 2024-11-26T07:53:52,623 INFO [RS:2;719ac8e87449:38357 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-26T07:53:52,623 INFO [RS:2;719ac8e87449:38357 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-26T07:53:52,623 INFO [RS:2;719ac8e87449:38357 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-26T07:53:52,623 INFO [RS:2;719ac8e87449:38357 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-26T07:53:52,624 INFO [RS:2;719ac8e87449:38357 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-26T07:53:52,624 INFO [RS:2;719ac8e87449:38357 {}] hbase.ChoreService(168): Chore ScheduledChore name=719ac8e87449,38357,1732607632486-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-26T07:53:52,645 INFO [RS:2;719ac8e87449:38357 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-26T07:53:52,646 INFO [RS:2;719ac8e87449:38357 {}] hbase.ChoreService(168): Chore ScheduledChore name=719ac8e87449,38357,1732607632486-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-26T07:53:52,646 INFO [RS:2;719ac8e87449:38357 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-26T07:53:52,646 INFO [RS:2;719ac8e87449:38357 {}] regionserver.Replication(171): 719ac8e87449,38357,1732607632486 started 2024-11-26T07:53:52,668 INFO [RS:2;719ac8e87449:38357 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-26T07:53:52,669 INFO [RS:2;719ac8e87449:38357 {}] regionserver.HRegionServer(1482): Serving as 719ac8e87449,38357,1732607632486, RpcServer on 719ac8e87449/172.17.0.2:38357, sessionid=0x101776be7e90003 2024-11-26T07:53:52,669 DEBUG [RS:2;719ac8e87449:38357 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-26T07:53:52,669 DEBUG [RS:2;719ac8e87449:38357 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 719ac8e87449,38357,1732607632486 2024-11-26T07:53:52,669 DEBUG [RS:2;719ac8e87449:38357 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '719ac8e87449,38357,1732607632486' 2024-11-26T07:53:52,669 DEBUG [RS:2;719ac8e87449:38357 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-26T07:53:52,670 DEBUG [RS:2;719ac8e87449:38357 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-26T07:53:52,670 DEBUG [RS:2;719ac8e87449:38357 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-26T07:53:52,670 DEBUG [RS:2;719ac8e87449:38357 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-26T07:53:52,671 DEBUG [RS:2;719ac8e87449:38357 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 719ac8e87449,38357,1732607632486 2024-11-26T07:53:52,671 DEBUG [RS:2;719ac8e87449:38357 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '719ac8e87449,38357,1732607632486' 2024-11-26T07:53:52,671 DEBUG [RS:2;719ac8e87449:38357 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-26T07:53:52,671 DEBUG [RS:2;719ac8e87449:38357 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-26T07:53:52,672 DEBUG [RS:2;719ac8e87449:38357 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-26T07:53:52,672 INFO [RS:2;719ac8e87449:38357 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-26T07:53:52,672 INFO [RS:2;719ac8e87449:38357 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-26T07:53:52,764 DEBUG [RSProcedureDispatcher-pool-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-26T07:53:52,766 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-8-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59507, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-26T07:53:52,769 INFO [RS_OPEN_META-regionserver/719ac8e87449:0-0 {event_type=M_RS_OPEN_META, pid=9}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-11-26T07:53:52,769 INFO [RS_OPEN_META-regionserver/719ac8e87449:0-0 {event_type=M_RS_OPEN_META, pid=9}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-26T07:53:52,770 INFO [RS_OPEN_META-regionserver/719ac8e87449:0-0 {event_type=M_RS_OPEN_META, pid=9}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-11-26T07:53:52,771 INFO [RS_OPEN_META-regionserver/719ac8e87449:0-0 {event_type=M_RS_OPEN_META, pid=9}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=719ac8e87449%2C34097%2C1732607628803.meta, suffix=.meta, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/480cd986-aef2-fe93-7892-18b5e497e91c/293d53df-6485-edb9-9b39-cf743ea8b2f2/WALs/719ac8e87449,34097,1732607628803, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/480cd986-aef2-fe93-7892-18b5e497e91c/293d53df-6485-edb9-9b39-cf743ea8b2f2/oldWALs, maxLogs=55 2024-11-26T07:53:52,773 INFO [RS:2;719ac8e87449:38357 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-26T07:53:52,774 INFO [RS:2;719ac8e87449:38357 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=719ac8e87449%2C38357%2C1732607632486, suffix=, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/480cd986-aef2-fe93-7892-18b5e497e91c/293d53df-6485-edb9-9b39-cf743ea8b2f2/WALs/719ac8e87449,38357,1732607632486, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/480cd986-aef2-fe93-7892-18b5e497e91c/293d53df-6485-edb9-9b39-cf743ea8b2f2/oldWALs, maxLogs=55 2024-11-26T07:53:52,823 INFO [RS_OPEN_META-regionserver/719ac8e87449:0-0 {event_type=M_RS_OPEN_META, pid=9}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/480cd986-aef2-fe93-7892-18b5e497e91c/293d53df-6485-edb9-9b39-cf743ea8b2f2/WALs/719ac8e87449,34097,1732607628803/719ac8e87449%2C34097%2C1732607628803.meta.1732607632772.meta 2024-11-26T07:53:52,826 DEBUG [RS_OPEN_META-regionserver/719ac8e87449:0-0 {event_type=M_RS_OPEN_META, pid=9}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-11-26T07:53:52,827 DEBUG [RS_OPEN_META-regionserver/719ac8e87449:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-11-26T07:53:52,827 DEBUG [RS_OPEN_META-regionserver/719ac8e87449:0-0 {event_type=M_RS_OPEN_META, pid=9}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-11-26T07:53:52,827 DEBUG [RS_OPEN_META-regionserver/719ac8e87449:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-11-26T07:53:52,827 INFO [RS_OPEN_META-regionserver/719ac8e87449:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-11-26T07:53:52,827 DEBUG [RS_OPEN_META-regionserver/719ac8e87449:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-11-26T07:53:52,828 DEBUG [RS_OPEN_META-regionserver/719ac8e87449:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-26T07:53:52,828 DEBUG [RS_OPEN_META-regionserver/719ac8e87449:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-11-26T07:53:52,828 DEBUG [RS_OPEN_META-regionserver/719ac8e87449:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-11-26T07:53:52,828 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family info of region 1588230740 2024-11-26T07:53:52,829 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-26T07:53:52,829 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T07:53:52,837 INFO [RS:2;719ac8e87449:38357 {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/480cd986-aef2-fe93-7892-18b5e497e91c/293d53df-6485-edb9-9b39-cf743ea8b2f2/WALs/719ac8e87449,38357,1732607632486/719ac8e87449%2C38357%2C1732607632486.1732607632775 2024-11-26T07:53:52,838 DEBUG [StoreFileOpener-1588230740-info-1 {}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/480cd986-aef2-fe93-7892-18b5e497e91c/293d53df-6485-edb9-9b39-cf743ea8b2f2/data/hbase/meta/1588230740/info/10e8dbaa8e7e42af8dad71e08a48806d, delay=955 ms 2024-11-26T07:53:52,838 DEBUG [RS:2;719ac8e87449:38357 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-11-26T07:53:52,840 DEBUG [StoreOpener-1588230740-1 {}] regionserver.StoreEngine(278): loaded file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/480cd986-aef2-fe93-7892-18b5e497e91c/293d53df-6485-edb9-9b39-cf743ea8b2f2/data/hbase/meta/1588230740/info/10e8dbaa8e7e42af8dad71e08a48806d 2024-11-26T07:53:52,840 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-26T07:53:52,841 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family ns of region 1588230740 2024-11-26T07:53:52,841 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-26T07:53:52,841 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T07:53:52,850 DEBUG [StoreFileOpener-1588230740-ns-1 {}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/480cd986-aef2-fe93-7892-18b5e497e91c/293d53df-6485-edb9-9b39-cf743ea8b2f2/data/hbase/meta/1588230740/ns/be72219bda5746729a3db47780310244, delay=976 ms 2024-11-26T07:53:52,851 DEBUG [StoreOpener-1588230740-1 {}] regionserver.StoreEngine(278): loaded file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/480cd986-aef2-fe93-7892-18b5e497e91c/293d53df-6485-edb9-9b39-cf743ea8b2f2/data/hbase/meta/1588230740/ns/be72219bda5746729a3db47780310244 2024-11-26T07:53:52,852 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-26T07:53:52,852 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family rep_barrier of region 1588230740 2024-11-26T07:53:52,853 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-26T07:53:52,853 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T07:53:52,853 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-26T07:53:52,853 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family table of region 1588230740 2024-11-26T07:53:52,854 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-26T07:53:52,854 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T07:53:52,861 DEBUG [StoreFileOpener-1588230740-table-1 {}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/480cd986-aef2-fe93-7892-18b5e497e91c/293d53df-6485-edb9-9b39-cf743ea8b2f2/data/hbase/meta/1588230740/table/46047b57f83540a88309c73fe3773775, delay=966 ms 2024-11-26T07:53:52,862 DEBUG [StoreOpener-1588230740-1 {}] regionserver.StoreEngine(278): loaded file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/480cd986-aef2-fe93-7892-18b5e497e91c/293d53df-6485-edb9-9b39-cf743ea8b2f2/data/hbase/meta/1588230740/table/46047b57f83540a88309c73fe3773775 2024-11-26T07:53:52,862 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-26T07:53:52,862 DEBUG [RS_OPEN_META-regionserver/719ac8e87449:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-26T07:53:52,863 DEBUG [RS_OPEN_META-regionserver/719ac8e87449:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/480cd986-aef2-fe93-7892-18b5e497e91c/293d53df-6485-edb9-9b39-cf743ea8b2f2/data/hbase/meta/1588230740 2024-11-26T07:53:52,864 DEBUG [RS_OPEN_META-regionserver/719ac8e87449:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/480cd986-aef2-fe93-7892-18b5e497e91c/293d53df-6485-edb9-9b39-cf743ea8b2f2/data/hbase/meta/1588230740 2024-11-26T07:53:52,864 DEBUG [RS_OPEN_META-regionserver/719ac8e87449:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-26T07:53:52,864 DEBUG [RS_OPEN_META-regionserver/719ac8e87449:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-26T07:53:52,865 DEBUG [RS_OPEN_META-regionserver/719ac8e87449:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-26T07:53:52,866 DEBUG [RS_OPEN_META-regionserver/719ac8e87449:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-26T07:53:52,867 INFO [RS_OPEN_META-regionserver/719ac8e87449:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=15; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=61616413, jitterRate=-0.08184389770030975}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-26T07:53:52,867 DEBUG [RS_OPEN_META-regionserver/719ac8e87449:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-11-26T07:53:52,868 DEBUG [RS_OPEN_META-regionserver/719ac8e87449:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1732607632828Writing region info on filesystem at 1732607632828Initializing all the Stores at 1732607632828Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732607632828Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732607632828Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732607632828Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732607632828Cleaning up temporary data from old regions at 1732607632864 (+36 ms)Running coprocessor post-open hooks at 1732607632867 (+3 ms)Region opened successfully at 1732607632868 (+1 ms) 2024-11-26T07:53:52,869 INFO [RS_OPEN_META-regionserver/719ac8e87449:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=9, masterSystemTime=1732607632763 2024-11-26T07:53:52,872 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=8 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=15, regionLocation=719ac8e87449,34097,1732607628803 2024-11-26T07:53:52,873 DEBUG [RS_OPEN_META-regionserver/719ac8e87449:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-11-26T07:53:52,873 INFO [PEWorker-1 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 719ac8e87449,34097,1732607628803, state=OPEN 2024-11-26T07:53:52,873 INFO [RS_OPEN_META-regionserver/719ac8e87449:0-0 {event_type=M_RS_OPEN_META, pid=9}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-11-26T07:53:52,890 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38357-0x101776be7e90003, quorum=127.0.0.1:64716, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-26T07:53:52,890 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36471-0x101776be7e90000, quorum=127.0.0.1:64716, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-26T07:53:52,890 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34097-0x101776be7e90002, quorum=127.0.0.1:64716, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-26T07:53:52,890 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-26T07:53:52,890 DEBUG [PEWorker-1 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=9, ppid=8, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=719ac8e87449,34097,1732607628803 2024-11-26T07:53:52,891 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-26T07:53:52,892 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-26T07:53:52,894 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=9, resume processing ppid=8 2024-11-26T07:53:52,894 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=9, ppid=8, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=719ac8e87449,34097,1732607628803 in 283 msec 2024-11-26T07:53:52,896 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=8, resume processing ppid=7 2024-11-26T07:53:52,896 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=8, ppid=7, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 450 msec 2024-11-26T07:53:52,897 INFO [PEWorker-5 {}] procedure.ServerCrashProcedure(207): 719ac8e87449,43013,1732607628743 had 2 regions 2024-11-26T07:53:52,897 INFO [PEWorker-5 {}] procedure.ServerCrashProcedure(339): Splitting WALs pid=7, state=RUNNABLE:SERVER_CRASH_SPLIT_LOGS, hasLock=true; ServerCrashProcedure 719ac8e87449,43013,1732607628743, splitWal=true, meta=true, isMeta: false 2024-11-26T07:53:52,898 INFO [PEWorker-5 {}] master.SplitLogManager(171): file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/480cd986-aef2-fe93-7892-18b5e497e91c/293d53df-6485-edb9-9b39-cf743ea8b2f2/WALs/719ac8e87449,43013,1732607628743-splitting dir is empty, no logs to split. 2024-11-26T07:53:52,898 INFO [PEWorker-5 {}] master.SplitWALManager(105): 719ac8e87449,43013,1732607628743 WAL count=0, meta=false 2024-11-26T07:53:52,899 INFO [PEWorker-5 {}] master.SplitLogManager(171): file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/480cd986-aef2-fe93-7892-18b5e497e91c/293d53df-6485-edb9-9b39-cf743ea8b2f2/WALs/719ac8e87449,43013,1732607628743-splitting dir is empty, no logs to split. 2024-11-26T07:53:52,899 INFO [PEWorker-5 {}] master.SplitWALManager(105): 719ac8e87449,43013,1732607628743 WAL count=0, meta=false 2024-11-26T07:53:52,899 DEBUG [PEWorker-5 {}] procedure.ServerCrashProcedure(329): Check if 719ac8e87449,43013,1732607628743 WAL splitting is done? wals=0, meta=false 2024-11-26T07:53:52,901 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=10, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1732607630266, region=21e7f52621ff432de1aac67fc922954b, ASSIGN}] 2024-11-26T07:53:52,902 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=10, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1732607630266, region=21e7f52621ff432de1aac67fc922954b, ASSIGN 2024-11-26T07:53:52,903 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=10, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1732607630266, region=21e7f52621ff432de1aac67fc922954b, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-11-26T07:53:53,053 DEBUG [719ac8e87449:36471 {}] balancer.BalancerClusterState(204): Hosts are {719ac8e87449=0} racks are {/default-rack=0} 2024-11-26T07:53:53,054 DEBUG [719ac8e87449:36471 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-11-26T07:53:53,054 DEBUG [719ac8e87449:36471 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-11-26T07:53:53,054 DEBUG [719ac8e87449:36471 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-11-26T07:53:53,054 DEBUG [719ac8e87449:36471 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-11-26T07:53:53,054 INFO [719ac8e87449:36471 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-11-26T07:53:53,054 INFO [719ac8e87449:36471 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-11-26T07:53:53,054 DEBUG [719ac8e87449:36471 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-11-26T07:53:53,054 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=10 updating hbase:meta row=21e7f52621ff432de1aac67fc922954b, regionState=OPENING, regionLocation=719ac8e87449,38357,1732607632486 2024-11-26T07:53:53,056 WARN [MiniHBaseClusterRegionServer-EventLoopGroup-8-2 {}] ipc.NettyRpcConnection$2(409): Exception encountered while connecting to the server 719ac8e87449:43013 org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 719ac8e87449/172.17.0.2:43013 Caused by: java.net.ConnectException: finishConnect(..) failed: Connection refused at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.newConnectException0(Errors.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.handleConnectErrno(Errors.java:131) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Socket.finishConnect(Socket.java:359) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.doFinishConnect(AbstractEpollChannel.java:715) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:692) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T07:53:53,057 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-8-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=719ac8e87449,43013,1732607628743, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=719ac8e87449,43013,1732607628743, seqNum=-1, error=java.net.ConnectException: Call to address=719ac8e87449:43013 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 719ac8e87449/172.17.0.2:43013 2024-11-26T07:53:53,057 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-8-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:meta,,1.1588230740, hostname=719ac8e87449,43013,1732607628743, seqNum=-1 is java.net.ConnectException: finishConnect(..) failed: Connection refused 2024-11-26T07:53:53,057 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-8-2 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:meta,,1.1588230740, hostname=719ac8e87449,43013,1732607628743, seqNum=-1 from cache 2024-11-26T07:53:53,058 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-8-2 {}] ipc.FailedServers(52): Added failed server with address 719ac8e87449:43013 to list caused by org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 719ac8e87449/172.17.0.2:43013 2024-11-26T07:53:53,167 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-26T07:53:53,167 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=719ac8e87449,34097,1732607628803, seqNum=-1] 2024-11-26T07:53:53,167 DEBUG [Async-Client-Retry-Timer-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-26T07:53:53,169 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster 2024-11-26T07:53:53,169 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-26T07:53:53,169 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-8-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53275, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-26T07:53:53,169 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79) at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611) at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.io.hfile.TestBlockEvictionOnRegionMovement.tearDown(TestBlockEvictionOnRegionMovement.java:174) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-26T07:53:53,169 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T07:53:53,170 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T07:53:53,170 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-26T07:53:53,170 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2024-11-26T07:53:53,170 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=945205514, stopped=false 2024-11-26T07:53:53,170 INFO [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=719ac8e87449,36471,1732607628597 2024-11-26T07:53:53,173 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-8-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=10, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1732607630266, region=21e7f52621ff432de1aac67fc922954b, ASSIGN because future has completed 2024-11-26T07:53:53,173 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=11, ppid=10, state=RUNNABLE, hasLock=false; OpenRegionProcedure 21e7f52621ff432de1aac67fc922954b, server=719ac8e87449,38357,1732607632486}] 2024-11-26T07:53:53,204 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38357-0x101776be7e90003, quorum=127.0.0.1:64716, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-26T07:53:53,204 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34097-0x101776be7e90002, quorum=127.0.0.1:64716, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-26T07:53:53,205 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36471-0x101776be7e90000, quorum=127.0.0.1:64716, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-26T07:53:53,205 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36471-0x101776be7e90000, quorum=127.0.0.1:64716, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-26T07:53:53,205 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-26T07:53:53,205 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34097-0x101776be7e90002, quorum=127.0.0.1:64716, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-26T07:53:53,205 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-26T07:53:53,205 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38357-0x101776be7e90003, quorum=127.0.0.1:64716, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-26T07:53:53,205 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277) at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265) at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416) at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.io.hfile.TestBlockEvictionOnRegionMovement.tearDown(TestBlockEvictionOnRegionMovement.java:174) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-26T07:53:53,205 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T07:53:53,206 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '719ac8e87449,34097,1732607628803' ***** 2024-11-26T07:53:53,206 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-11-26T07:53:53,206 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '719ac8e87449,38357,1732607632486' ***** 2024-11-26T07:53:53,206 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-11-26T07:53:53,206 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:38357-0x101776be7e90003, quorum=127.0.0.1:64716, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-26T07:53:53,206 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:34097-0x101776be7e90002, quorum=127.0.0.1:64716, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-26T07:53:53,206 INFO [RS:2;719ac8e87449:38357 {}] bucket.BucketCache(1738): Disabling cache 2024-11-26T07:53:53,206 INFO [RS:1;719ac8e87449:34097 {}] bucket.BucketCache(1738): Disabling cache 2024-11-26T07:53:53,207 INFO [RS:1;719ac8e87449:34097 {}] bucket.BucketCache(1762): Shutdown bucket cache: IO persistent=true; path to write=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3acb3e4a-dea5-ea8d-f2d0-bb92c503e71b/bucket.persistence 2024-11-26T07:53:53,207 INFO [Time-limited test-BucketCacheWriter-0 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-0 exiting, cacheEnabled=false 2024-11-26T07:53:53,207 INFO [Time-limited test-BucketCacheWriter-2 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-2 exiting, cacheEnabled=false 2024-11-26T07:53:53,207 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:36471-0x101776be7e90000, quorum=127.0.0.1:64716, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-26T07:53:53,207 INFO [RS:2;719ac8e87449:38357 {}] bucket.BucketCache(1762): Shutdown bucket cache: IO persistent=true; path to write=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3acb3e4a-dea5-ea8d-f2d0-bb92c503e71b/bucket.persistence 2024-11-26T07:53:53,207 INFO [Time-limited test-BucketCacheWriter-2 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-2 exiting, cacheEnabled=false 2024-11-26T07:53:53,207 INFO [Time-limited test-BucketCacheWriter-1 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-1 exiting, cacheEnabled=false 2024-11-26T07:53:53,207 INFO [Time-limited test-BucketCacheWriter-1 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-1 exiting, cacheEnabled=false 2024-11-26T07:53:53,207 INFO [Time-limited test-BucketCacheWriter-0 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-0 exiting, cacheEnabled=false 2024-11-26T07:53:53,207 INFO [RS:1;719ac8e87449:34097 {}] bucket.BucketCache(1768): Shutting down cache persister thread. 2024-11-26T07:53:53,207 INFO [RS:2;719ac8e87449:38357 {}] bucket.BucketCache(1768): Shutting down cache persister thread. 2024-11-26T07:53:53,207 WARN [bucket-cache-persister {}] bucket.BucketCachePersister(64): Interrupting BucketCachePersister thread. java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCachePersister.run(BucketCachePersister.java:45) ~[classes/:?] 2024-11-26T07:53:53,207 WARN [bucket-cache-persister {}] bucket.BucketCachePersister(64): Interrupting BucketCachePersister thread. java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCachePersister.run(BucketCachePersister.java:45) ~[classes/:?] 2024-11-26T07:53:53,217 DEBUG [RS:1;719ac8e87449:34097 {}] bucket.BucketCache(1487): Thread RS:1;719ac8e87449:34097 started persisting bucket cache to file 2024-11-26T07:53:53,217 DEBUG [RS:2;719ac8e87449:38357 {}] bucket.BucketCache(1487): Thread RS:2;719ac8e87449:38357 started persisting bucket cache to file 2024-11-26T07:53:53,218 DEBUG [RS:1;719ac8e87449:34097 {}] bucket.BucketCache(1494): Persist in new chunked persistence format. 2024-11-26T07:53:53,218 DEBUG [RS:2;719ac8e87449:38357 {}] bucket.BucketCache(1494): Persist in new chunked persistence format. 2024-11-26T07:53:53,218 DEBUG [RS:1;719ac8e87449:34097 {}] bucket.BucketCache(1673): persistToFile: before persisting backing map size: 0, fullycachedFiles size: 0, chunkSize: 10000 2024-11-26T07:53:53,218 DEBUG [RS:2;719ac8e87449:38357 {}] bucket.BucketCache(1673): persistToFile: before persisting backing map size: 0, fullycachedFiles size: 0, chunkSize: 10000 2024-11-26T07:53:53,219 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-26T07:53:53,221 DEBUG [RS:2;719ac8e87449:38357 {}] bucket.BucketCache(1680): persistToFile: after persisting backing map size: 0, fullycachedFiles size: 0 2024-11-26T07:53:53,221 DEBUG [RS:2;719ac8e87449:38357 {}] bucket.BucketCache(1498): PersistToFile: after persisting backing map size: 0, fullycachedFiles size: 0, file name: bucket.persistence1732607633217 2024-11-26T07:53:53,221 WARN [Thread-217 {}] util.Shell$1(1036): Error reading the error stream java.io.IOException: Stream closed at java.io.BufferedInputStream.getBufIfOpen(BufferedInputStream.java:168) ~[?:?] at java.io.BufferedInputStream.read(BufferedInputStream.java:334) ~[?:?] at sun.nio.cs.StreamDecoder.readBytes(StreamDecoder.java:281) ~[?:?] at sun.nio.cs.StreamDecoder.implRead(StreamDecoder.java:324) ~[?:?] at sun.nio.cs.StreamDecoder.read(StreamDecoder.java:189) ~[?:?] at java.io.InputStreamReader.read(InputStreamReader.java:177) ~[?:?] at java.io.BufferedReader.fill(BufferedReader.java:162) ~[?:?] at java.io.BufferedReader.readLine(BufferedReader.java:329) ~[?:?] at java.io.BufferedReader.readLine(BufferedReader.java:396) ~[?:?] at org.apache.hadoop.util.Shell$1.run(Shell.java:1025) ~[hadoop-common-3.4.1.jar:?] 2024-11-26T07:53:53,222 DEBUG [RS:1;719ac8e87449:34097 {}] bucket.BucketCache(1680): persistToFile: after persisting backing map size: 0, fullycachedFiles size: 0 2024-11-26T07:53:53,222 DEBUG [RS:1;719ac8e87449:34097 {}] bucket.BucketCache(1498): PersistToFile: after persisting backing map size: 0, fullycachedFiles size: 0, file name: bucket.persistence1732607633217 2024-11-26T07:53:53,222 DEBUG [RS:1;719ac8e87449:34097 {}] bucket.BucketCache(1509): Thread RS:1;719ac8e87449:34097 finished persisting bucket cache to file, renaming 2024-11-26T07:53:53,222 INFO [RS:1;719ac8e87449:34097 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-26T07:53:53,222 INFO [RS:1;719ac8e87449:34097 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-26T07:53:53,222 INFO [RS:1;719ac8e87449:34097 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-26T07:53:53,222 INFO [RS:1;719ac8e87449:34097 {}] regionserver.HRegionServer(959): stopping server 719ac8e87449,34097,1732607628803 2024-11-26T07:53:53,222 INFO [RS:1;719ac8e87449:34097 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-26T07:53:53,222 INFO [RS:1;719ac8e87449:34097 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:1;719ac8e87449:34097. 2024-11-26T07:53:53,223 DEBUG [RS:1;719ac8e87449:34097 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-26T07:53:53,223 DEBUG [RS:1;719ac8e87449:34097 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T07:53:53,223 INFO [RS:1;719ac8e87449:34097 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-26T07:53:53,223 INFO [RS:1;719ac8e87449:34097 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-26T07:53:53,223 INFO [RS:1;719ac8e87449:34097 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-26T07:53:53,223 INFO [regionserver/719ac8e87449:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-26T07:53:53,223 INFO [RS:1;719ac8e87449:34097 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-11-26T07:53:53,223 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-26T07:53:53,223 INFO [RS:1;719ac8e87449:34097 {}] regionserver.HRegionServer(1321): Waiting on 1 regions to close 2024-11-26T07:53:53,223 DEBUG [RS:1;719ac8e87449:34097 {}] regionserver.HRegionServer(1325): Online Regions={1588230740=hbase:meta,,1.1588230740} 2024-11-26T07:53:53,223 DEBUG [RS:1;719ac8e87449:34097 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-26T07:53:53,224 INFO [RS_CLOSE_META-regionserver/719ac8e87449:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1662): Closing region hbase:meta,,1.1588230740 during a graceful stop, and cache persistence is on, so setting evict on close to false. 2024-11-26T07:53:53,224 DEBUG [RS_CLOSE_META-regionserver/719ac8e87449:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-26T07:53:53,224 INFO [RS_CLOSE_META-regionserver/719ac8e87449:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-26T07:53:53,224 DEBUG [RS_CLOSE_META-regionserver/719ac8e87449:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-26T07:53:53,224 DEBUG [RS_CLOSE_META-regionserver/719ac8e87449:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-26T07:53:53,224 DEBUG [RS_CLOSE_META-regionserver/719ac8e87449:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-26T07:53:53,224 INFO [RS_CLOSE_META-regionserver/719ac8e87449:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=485 B heapSize=1.70 KB 2024-11-26T07:53:53,225 INFO [regionserver/719ac8e87449:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-26T07:53:53,237 DEBUG [RS:2;719ac8e87449:38357 {}] bucket.BucketCache(1509): Thread RS:2;719ac8e87449:38357 finished persisting bucket cache to file, renaming 2024-11-26T07:53:53,237 WARN [RS:2;719ac8e87449:38357 {}] bucket.BucketCache(1512): Failed to commit cache persistent file. We might lose cached blocks if RS crashes/restarts before we successfully checkpoint again. 2024-11-26T07:53:53,237 INFO [RS:2;719ac8e87449:38357 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-26T07:53:53,238 INFO [RS:2;719ac8e87449:38357 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-26T07:53:53,238 INFO [RS:2;719ac8e87449:38357 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-26T07:53:53,238 INFO [RS:2;719ac8e87449:38357 {}] regionserver.HRegionServer(959): stopping server 719ac8e87449,38357,1732607632486 2024-11-26T07:53:53,238 INFO [RS:2;719ac8e87449:38357 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-26T07:53:53,238 INFO [RS:2;719ac8e87449:38357 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:2;719ac8e87449:38357. 2024-11-26T07:53:53,238 DEBUG [RS:2;719ac8e87449:38357 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-26T07:53:53,238 DEBUG [RS:2;719ac8e87449:38357 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T07:53:53,238 INFO [RS:2;719ac8e87449:38357 {}] regionserver.HRegionServer(976): stopping server 719ac8e87449,38357,1732607632486; all regions closed. 2024-11-26T07:53:53,242 DEBUG [RS:2;719ac8e87449:38357 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/480cd986-aef2-fe93-7892-18b5e497e91c/293d53df-6485-edb9-9b39-cf743ea8b2f2/oldWALs 2024-11-26T07:53:53,242 INFO [RS:2;719ac8e87449:38357 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 719ac8e87449%2C38357%2C1732607632486:(num 1732607632775) 2024-11-26T07:53:53,242 DEBUG [RS:2;719ac8e87449:38357 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T07:53:53,242 INFO [RS:2;719ac8e87449:38357 {}] regionserver.LeaseManager(133): Closed leases 2024-11-26T07:53:53,243 INFO [RS:2;719ac8e87449:38357 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-26T07:53:53,243 INFO [RS:2;719ac8e87449:38357 {}] hbase.ChoreService(370): Chore service for: regionserver/719ac8e87449:0 had [ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-11-26T07:53:53,244 INFO [RS:2;719ac8e87449:38357 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-26T07:53:53,244 INFO [RS:2;719ac8e87449:38357 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-26T07:53:53,244 INFO [RS:2;719ac8e87449:38357 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-26T07:53:53,244 INFO [RS:2;719ac8e87449:38357 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-26T07:53:53,244 INFO [RS:2;719ac8e87449:38357 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:38357 2024-11-26T07:53:53,244 INFO [regionserver/719ac8e87449:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-26T07:53:53,247 DEBUG [RS_CLOSE_META-regionserver/719ac8e87449:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 51e427366b1c4a448876e33500ad0bc1 2024-11-26T07:53:53,266 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36471-0x101776be7e90000, quorum=127.0.0.1:64716, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-26T07:53:53,266 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38357-0x101776be7e90003, quorum=127.0.0.1:64716, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/719ac8e87449,38357,1732607632486 2024-11-26T07:53:53,266 INFO [RS:2;719ac8e87449:38357 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-26T07:53:53,267 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [719ac8e87449,38357,1732607632486] 2024-11-26T07:53:53,267 DEBUG [RS_CLOSE_META-regionserver/719ac8e87449:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/480cd986-aef2-fe93-7892-18b5e497e91c/293d53df-6485-edb9-9b39-cf743ea8b2f2/data/hbase/meta/1588230740/.tmp/info/51e427366b1c4a448876e33500ad0bc1 is 209, key is testBlockEvictionOnGracefulStop1732607630266,,1732607630270.21e7f52621ff432de1aac67fc922954b./info:regioninfo/1732607633054/Put/seqid=0 2024-11-26T07:53:53,268 INFO [RS_CLOSE_META-regionserver/719ac8e87449:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=485 B at sequenceid=18 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/480cd986-aef2-fe93-7892-18b5e497e91c/293d53df-6485-edb9-9b39-cf743ea8b2f2/data/hbase/meta/1588230740/.tmp/info/51e427366b1c4a448876e33500ad0bc1 2024-11-26T07:53:53,270 DEBUG [RS_CLOSE_META-regionserver/719ac8e87449:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/480cd986-aef2-fe93-7892-18b5e497e91c/293d53df-6485-edb9-9b39-cf743ea8b2f2/data/hbase/meta/1588230740/.tmp/info/51e427366b1c4a448876e33500ad0bc1 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/480cd986-aef2-fe93-7892-18b5e497e91c/293d53df-6485-edb9-9b39-cf743ea8b2f2/data/hbase/meta/1588230740/info/51e427366b1c4a448876e33500ad0bc1 2024-11-26T07:53:53,272 DEBUG [RS_CLOSE_META-regionserver/719ac8e87449:0-0 {event_type=M_RS_CLOSE_META}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/480cd986-aef2-fe93-7892-18b5e497e91c/293d53df-6485-edb9-9b39-cf743ea8b2f2/data/hbase/meta/1588230740/info/51e427366b1c4a448876e33500ad0bc1, delay=901 ms 2024-11-26T07:53:53,273 INFO [RS_CLOSE_META-regionserver/719ac8e87449:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/480cd986-aef2-fe93-7892-18b5e497e91c/293d53df-6485-edb9-9b39-cf743ea8b2f2/data/hbase/meta/1588230740/info/51e427366b1c4a448876e33500ad0bc1, entries=3, sequenceid=18, filesize=5.9 K 2024-11-26T07:53:53,274 INFO [RS_CLOSE_META-regionserver/719ac8e87449:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~485 B/485, heapSize ~952 B/952, currentSize=0 B/0 for 1588230740 in 50ms, sequenceid=18, compaction requested=false 2024-11-26T07:53:53,275 DEBUG [StoreFileCloser-1588230740-info-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/480cd986-aef2-fe93-7892-18b5e497e91c/293d53df-6485-edb9-9b39-cf743ea8b2f2/data/hbase/meta/1588230740/info/10e8dbaa8e7e42af8dad71e08a48806d 2024-11-26T07:53:53,275 DEBUG [StoreFileCloser-1588230740-info-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/480cd986-aef2-fe93-7892-18b5e497e91c/293d53df-6485-edb9-9b39-cf743ea8b2f2/data/hbase/meta/1588230740/info/51e427366b1c4a448876e33500ad0bc1 2024-11-26T07:53:53,275 DEBUG [StoreFileCloser-1588230740-ns-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/480cd986-aef2-fe93-7892-18b5e497e91c/293d53df-6485-edb9-9b39-cf743ea8b2f2/data/hbase/meta/1588230740/ns/be72219bda5746729a3db47780310244 2024-11-26T07:53:53,276 DEBUG [StoreFileCloser-1588230740-table-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/480cd986-aef2-fe93-7892-18b5e497e91c/293d53df-6485-edb9-9b39-cf743ea8b2f2/data/hbase/meta/1588230740/table/46047b57f83540a88309c73fe3773775 2024-11-26T07:53:53,282 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/719ac8e87449,38357,1732607632486 already deleted, retry=false 2024-11-26T07:53:53,282 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; 719ac8e87449,38357,1732607632486 expired; onlineServers=1 2024-11-26T07:53:53,287 DEBUG [RS_CLOSE_META-regionserver/719ac8e87449:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/480cd986-aef2-fe93-7892-18b5e497e91c/293d53df-6485-edb9-9b39-cf743ea8b2f2/data/hbase/meta/1588230740/recovered.edits/21.seqid, newMaxSeqId=21, maxSeqId=14 2024-11-26T07:53:53,287 DEBUG [RS_CLOSE_META-regionserver/719ac8e87449:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-26T07:53:53,287 INFO [RS_CLOSE_META-regionserver/719ac8e87449:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-26T07:53:53,287 DEBUG [RS_CLOSE_META-regionserver/719ac8e87449:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1732607633224Running coprocessor pre-close hooks at 1732607633224Disabling compacts and flushes for region at 1732607633224Disabling writes for close at 1732607633224Obtaining lock to block concurrent updates at 1732607633224Preparing flush snapshotting stores in 1588230740 at 1732607633224Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=485, getHeapSize=1672, getOffHeapSize=0, getCellsCount=3 at 1732607633225 (+1 ms)Flushing stores of hbase:meta,,1.1588230740 at 1732607633225Flushing 1588230740/info: creating writer at 1732607633225Flushing 1588230740/info: appending metadata at 1732607633266 (+41 ms)Flushing 1588230740/info: closing flushed file at 1732607633266Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@4dd6710b: reopening flushed file at 1732607633270 (+4 ms)Finished flush of dataSize ~485 B/485, heapSize ~952 B/952, currentSize=0 B/0 for 1588230740 in 50ms, sequenceid=18, compaction requested=false at 1732607633274 (+4 ms)Writing region close event to WAL at 1732607633276 (+2 ms)Running coprocessor post-close hooks at 1732607633287 (+11 ms)Closed at 1732607633287 2024-11-26T07:53:53,287 DEBUG [RS_CLOSE_META-regionserver/719ac8e87449:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-11-26T07:53:53,327 DEBUG [RSProcedureDispatcher-pool-0 {}] assignment.RegionRemoteProcedureBase(128): pid=11, ppid=10, state=RUNNABLE, hasLock=false; OpenRegionProcedure 21e7f52621ff432de1aac67fc922954b, server=719ac8e87449,38357,1732607632486 for region state=OPENING, location=719ac8e87449,38357,1732607632486, table=testBlockEvictionOnGracefulStop1732607630266, region=21e7f52621ff432de1aac67fc922954b, targetServer 719ac8e87449,38357,1732607632486 is dead, SCP will interrupt us, give up 2024-11-26T07:53:53,390 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38357-0x101776be7e90003, quorum=127.0.0.1:64716, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-26T07:53:53,390 INFO [RS:2;719ac8e87449:38357 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-26T07:53:53,390 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38357-0x101776be7e90003, quorum=127.0.0.1:64716, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-26T07:53:53,390 INFO [RS:2;719ac8e87449:38357 {}] regionserver.HRegionServer(1031): Exiting; stopping=719ac8e87449,38357,1732607632486; zookeeper connection closed. 2024-11-26T07:53:53,391 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@53b47f30 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@53b47f30 2024-11-26T07:53:53,424 INFO [RS:1;719ac8e87449:34097 {}] regionserver.HRegionServer(976): stopping server 719ac8e87449,34097,1732607628803; all regions closed. 2024-11-26T07:53:53,426 DEBUG [RS:1;719ac8e87449:34097 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/480cd986-aef2-fe93-7892-18b5e497e91c/293d53df-6485-edb9-9b39-cf743ea8b2f2/oldWALs 2024-11-26T07:53:53,426 INFO [RS:1;719ac8e87449:34097 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 719ac8e87449%2C34097%2C1732607628803.meta:.meta(num 1732607632772) 2024-11-26T07:53:53,430 DEBUG [RS:1;719ac8e87449:34097 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/480cd986-aef2-fe93-7892-18b5e497e91c/293d53df-6485-edb9-9b39-cf743ea8b2f2/oldWALs 2024-11-26T07:53:53,430 INFO [RS:1;719ac8e87449:34097 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 719ac8e87449%2C34097%2C1732607628803:(num 1732607629662) 2024-11-26T07:53:53,430 DEBUG [RS:1;719ac8e87449:34097 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T07:53:53,430 INFO [RS:1;719ac8e87449:34097 {}] regionserver.LeaseManager(133): Closed leases 2024-11-26T07:53:53,430 INFO [RS:1;719ac8e87449:34097 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-26T07:53:53,430 INFO [RS:1;719ac8e87449:34097 {}] hbase.ChoreService(370): Chore service for: regionserver/719ac8e87449:0 had [ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-11-26T07:53:53,431 INFO [RS:1;719ac8e87449:34097 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-26T07:53:53,431 INFO [regionserver/719ac8e87449:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-26T07:53:53,431 INFO [RS:1;719ac8e87449:34097 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:34097 2024-11-26T07:53:53,448 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36471-0x101776be7e90000, quorum=127.0.0.1:64716, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-26T07:53:53,448 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34097-0x101776be7e90002, quorum=127.0.0.1:64716, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/719ac8e87449,34097,1732607628803 2024-11-26T07:53:53,448 INFO [RS:1;719ac8e87449:34097 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-26T07:53:53,449 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [719ac8e87449,34097,1732607628803] 2024-11-26T07:53:53,465 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/719ac8e87449,34097,1732607628803 already deleted, retry=false 2024-11-26T07:53:53,465 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; 719ac8e87449,34097,1732607628803 expired; onlineServers=0 2024-11-26T07:53:53,465 INFO [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master '719ac8e87449,36471,1732607628597' ***** 2024-11-26T07:53:53,465 INFO [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0 2024-11-26T07:53:53,465 INFO [M:0;719ac8e87449:36471 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-26T07:53:53,465 INFO [M:0;719ac8e87449:36471 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-26T07:53:53,465 DEBUG [M:0;719ac8e87449:36471 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2024-11-26T07:53:53,466 DEBUG [M:0;719ac8e87449:36471 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2024-11-26T07:53:53,466 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2024-11-26T07:53:53,466 DEBUG [master/719ac8e87449:0:becomeActiveMaster-HFileCleaner.small.0-1732607629314 {}] cleaner.HFileCleaner(306): Exit Thread[master/719ac8e87449:0:becomeActiveMaster-HFileCleaner.small.0-1732607629314,5,FailOnTimeoutGroup] 2024-11-26T07:53:53,466 DEBUG [master/719ac8e87449:0:becomeActiveMaster-HFileCleaner.large.0-1732607629313 {}] cleaner.HFileCleaner(306): Exit Thread[master/719ac8e87449:0:becomeActiveMaster-HFileCleaner.large.0-1732607629313,5,FailOnTimeoutGroup] 2024-11-26T07:53:53,466 INFO [M:0;719ac8e87449:36471 {}] hbase.ChoreService(370): Chore service for: master/719ac8e87449:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown 2024-11-26T07:53:53,466 INFO [M:0;719ac8e87449:36471 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-26T07:53:53,466 DEBUG [M:0;719ac8e87449:36471 {}] master.HMaster(1795): Stopping service threads 2024-11-26T07:53:53,466 INFO [M:0;719ac8e87449:36471 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2024-11-26T07:53:53,466 INFO [M:0;719ac8e87449:36471 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-26T07:53:53,466 INFO [M:0;719ac8e87449:36471 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-26T07:53:53,466 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2024-11-26T07:53:53,474 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36471-0x101776be7e90000, quorum=127.0.0.1:64716, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-11-26T07:53:53,474 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36471-0x101776be7e90000, quorum=127.0.0.1:64716, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-26T07:53:53,474 DEBUG [M:0;719ac8e87449:36471 {}] zookeeper.ZKUtil(347): master:36471-0x101776be7e90000, quorum=127.0.0.1:64716, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error) 2024-11-26T07:53:53,474 WARN [M:0;719ac8e87449:36471 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null 2024-11-26T07:53:53,474 INFO [M:0;719ac8e87449:36471 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/480cd986-aef2-fe93-7892-18b5e497e91c/293d53df-6485-edb9-9b39-cf743ea8b2f2/.lastflushedseqids 2024-11-26T07:53:53,480 INFO [M:0;719ac8e87449:36471 {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-26T07:53:53,480 INFO [M:0;719ac8e87449:36471 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2024-11-26T07:53:53,480 DEBUG [M:0;719ac8e87449:36471 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-26T07:53:53,480 INFO [M:0;719ac8e87449:36471 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-26T07:53:53,480 DEBUG [M:0;719ac8e87449:36471 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-26T07:53:53,480 DEBUG [M:0;719ac8e87449:36471 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-26T07:53:53,480 DEBUG [M:0;719ac8e87449:36471 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-26T07:53:53,481 INFO [M:0;719ac8e87449:36471 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=34.95 KB heapSize=44.17 KB 2024-11-26T07:53:53,490 DEBUG [M:0;719ac8e87449:36471 {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 8855aa87941947aaa65729c2c28a7ec1 2024-11-26T07:53:53,508 DEBUG [M:0;719ac8e87449:36471 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/480cd986-aef2-fe93-7892-18b5e497e91c/293d53df-6485-edb9-9b39-cf743ea8b2f2/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/8855aa87941947aaa65729c2c28a7ec1 is 82, key is hbase:meta,,1/info:regioninfo/1732607632872/Put/seqid=0 2024-11-26T07:53:53,509 INFO [M:0;719ac8e87449:36471 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1008 B at sequenceid=90 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/480cd986-aef2-fe93-7892-18b5e497e91c/293d53df-6485-edb9-9b39-cf743ea8b2f2/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/8855aa87941947aaa65729c2c28a7ec1 2024-11-26T07:53:53,517 DEBUG [M:0;719ac8e87449:36471 {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 2d35d19f414b411b81e00aebdaa3aeaa 2024-11-26T07:53:53,530 DEBUG [M:0;719ac8e87449:36471 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/480cd986-aef2-fe93-7892-18b5e497e91c/293d53df-6485-edb9-9b39-cf743ea8b2f2/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/2d35d19f414b411b81e00aebdaa3aeaa is 783, key is \x00\x00\x00\x00\x00\x00\x00\x04/proc:d/1732607630693/Put/seqid=0 2024-11-26T07:53:53,531 INFO [M:0;719ac8e87449:36471 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.73 KB at sequenceid=90 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/480cd986-aef2-fe93-7892-18b5e497e91c/293d53df-6485-edb9-9b39-cf743ea8b2f2/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/2d35d19f414b411b81e00aebdaa3aeaa 2024-11-26T07:53:53,538 DEBUG [M:0;719ac8e87449:36471 {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on cc613d58fbab4f92a90be8c0f2edae87 2024-11-26T07:53:53,557 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34097-0x101776be7e90002, quorum=127.0.0.1:64716, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-26T07:53:53,557 INFO [RS:1;719ac8e87449:34097 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-26T07:53:53,557 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34097-0x101776be7e90002, quorum=127.0.0.1:64716, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-26T07:53:53,557 INFO [RS:1;719ac8e87449:34097 {}] regionserver.HRegionServer(1031): Exiting; stopping=719ac8e87449,34097,1732607628803; zookeeper connection closed. 2024-11-26T07:53:53,557 DEBUG [M:0;719ac8e87449:36471 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/480cd986-aef2-fe93-7892-18b5e497e91c/293d53df-6485-edb9-9b39-cf743ea8b2f2/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/cc613d58fbab4f92a90be8c0f2edae87 is 69, key is 719ac8e87449,34097,1732607628803/rs:state/1732607629454/Put/seqid=0 2024-11-26T07:53:53,557 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@7185054d {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@7185054d 2024-11-26T07:53:53,558 INFO [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 3 regionserver(s) complete 2024-11-26T07:53:53,558 INFO [M:0;719ac8e87449:36471 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=249 B at sequenceid=90 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/480cd986-aef2-fe93-7892-18b5e497e91c/293d53df-6485-edb9-9b39-cf743ea8b2f2/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/cc613d58fbab4f92a90be8c0f2edae87 2024-11-26T07:53:53,560 INFO [M:0;719ac8e87449:36471 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for cc613d58fbab4f92a90be8c0f2edae87 2024-11-26T07:53:53,561 DEBUG [M:0;719ac8e87449:36471 {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/480cd986-aef2-fe93-7892-18b5e497e91c/293d53df-6485-edb9-9b39-cf743ea8b2f2/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/8855aa87941947aaa65729c2c28a7ec1 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/480cd986-aef2-fe93-7892-18b5e497e91c/293d53df-6485-edb9-9b39-cf743ea8b2f2/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/8855aa87941947aaa65729c2c28a7ec1 2024-11-26T07:53:53,563 INFO [M:0;719ac8e87449:36471 {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/480cd986-aef2-fe93-7892-18b5e497e91c/293d53df-6485-edb9-9b39-cf743ea8b2f2/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/8855aa87941947aaa65729c2c28a7ec1, entries=14, sequenceid=90, filesize=5.9 K 2024-11-26T07:53:53,563 DEBUG [M:0;719ac8e87449:36471 {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/480cd986-aef2-fe93-7892-18b5e497e91c/293d53df-6485-edb9-9b39-cf743ea8b2f2/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/2d35d19f414b411b81e00aebdaa3aeaa as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/480cd986-aef2-fe93-7892-18b5e497e91c/293d53df-6485-edb9-9b39-cf743ea8b2f2/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/2d35d19f414b411b81e00aebdaa3aeaa 2024-11-26T07:53:53,565 INFO [M:0;719ac8e87449:36471 {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/480cd986-aef2-fe93-7892-18b5e497e91c/293d53df-6485-edb9-9b39-cf743ea8b2f2/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/2d35d19f414b411b81e00aebdaa3aeaa, entries=11, sequenceid=90, filesize=7.8 K 2024-11-26T07:53:53,566 DEBUG [M:0;719ac8e87449:36471 {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/480cd986-aef2-fe93-7892-18b5e497e91c/293d53df-6485-edb9-9b39-cf743ea8b2f2/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/cc613d58fbab4f92a90be8c0f2edae87 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/480cd986-aef2-fe93-7892-18b5e497e91c/293d53df-6485-edb9-9b39-cf743ea8b2f2/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/cc613d58fbab4f92a90be8c0f2edae87 2024-11-26T07:53:53,568 INFO [M:0;719ac8e87449:36471 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for cc613d58fbab4f92a90be8c0f2edae87 2024-11-26T07:53:53,568 INFO [M:0;719ac8e87449:36471 {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/480cd986-aef2-fe93-7892-18b5e497e91c/293d53df-6485-edb9-9b39-cf743ea8b2f2/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/cc613d58fbab4f92a90be8c0f2edae87, entries=3, sequenceid=90, filesize=5.3 K 2024-11-26T07:53:53,569 INFO [M:0;719ac8e87449:36471 {}] regionserver.HRegion(3140): Finished flush of dataSize ~34.95 KB/35792, heapSize ~43.88 KB/44928, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 89ms, sequenceid=90, compaction requested=false 2024-11-26T07:53:53,572 INFO [M:0;719ac8e87449:36471 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-26T07:53:53,572 DEBUG [M:0;719ac8e87449:36471 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1732607633480Disabling compacts and flushes for region at 1732607633480Disabling writes for close at 1732607633480Obtaining lock to block concurrent updates at 1732607633481 (+1 ms)Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1732607633481Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=35792, getHeapSize=45168, getOffHeapSize=0, getCellsCount=111 at 1732607633481Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1732607633482 (+1 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1732607633482Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1732607633507 (+25 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1732607633507Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1732607633511 (+4 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1732607633529 (+18 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1732607633529Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1732607633532 (+3 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1732607633556 (+24 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1732607633556Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@6ac7885d: reopening flushed file at 1732607633560 (+4 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@3d3ad7d7: reopening flushed file at 1732607633563 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@2bdf9ab2: reopening flushed file at 1732607633565 (+2 ms)Finished flush of dataSize ~34.95 KB/35792, heapSize ~43.88 KB/44928, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 89ms, sequenceid=90, compaction requested=false at 1732607633569 (+4 ms)Writing region close event to WAL at 1732607633571 (+2 ms)Closed at 1732607633571 2024-11-26T07:53:53,573 INFO [M:0;719ac8e87449:36471 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down. 2024-11-26T07:53:53,573 INFO [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-26T07:53:53,573 INFO [M:0;719ac8e87449:36471 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:36471 2024-11-26T07:53:53,573 INFO [M:0;719ac8e87449:36471 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-26T07:53:53,704 INFO [M:0;719ac8e87449:36471 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-26T07:53:53,704 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36471-0x101776be7e90000, quorum=127.0.0.1:64716, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-26T07:53:53,705 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36471-0x101776be7e90000, quorum=127.0.0.1:64716, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-26T07:53:53,708 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers 2024-11-26T07:53:53,709 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down 2024-11-26T07:53:53,709 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-26T07:53:53,716 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: io.hfile.TestBlockEvictionOnRegionMovement#testBlockEvictionOnGracefulStop Thread=71 (was 49) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-8-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-9-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-6 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-9-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-5 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-8-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HMaster-EventLoopGroup-6-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-7-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-8 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ForkJoinPool.commonPool-worker-4 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.ForkJoinPool.awaitWork(ForkJoinPool.java:1724) java.base@17.0.11/java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1623) java.base@17.0.11/java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:165) Potentially hanging thread: HMaster-EventLoopGroup-6-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-7-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-8-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-7-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-7 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HMaster-EventLoopGroup-6-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=382 (was 346) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=426 (was 428), ProcessCount=11 (was 11), AvailableMemoryMB=10889 (was 10969)