2024-11-16 14:56:58,004 main DEBUG Apache Log4j Core 2.17.2 initializing configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@15a34df2 2024-11-16 14:56:58,021 main DEBUG Took 0.014974 seconds to load 1 plugins from package org.apache.hadoop.hbase.logging 2024-11-16 14:56:58,021 main DEBUG PluginManager 'Core' found 129 plugins 2024-11-16 14:56:58,021 main DEBUG PluginManager 'Level' found 0 plugins 2024-11-16 14:56:58,022 main DEBUG PluginManager 'Lookup' found 16 plugins 2024-11-16 14:56:58,023 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-16 14:56:58,030 main DEBUG PluginManager 'TypeConverter' found 26 plugins 2024-11-16 14:56:58,041 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.util.MBeans", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-16 14:56:58,043 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-16 14:56:58,043 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.logging.TestJul2Slf4j", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-16 14:56:58,044 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-16 14:56:58,044 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.zookeeper", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-16 14:56:58,044 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-16 14:56:58,045 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSinkAdapter", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-16 14:56:58,045 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-16 14:56:58,046 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSystemImpl", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-16 14:56:58,046 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-16 14:56:58,047 main DEBUG LoggerConfig$Builder(additivity="false", level="WARN", levelAndRefs="null", name="org.apache.directory", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-16 14:56:58,047 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-16 14:56:58,048 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.ipc.FailedServers", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-16 14:56:58,048 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-16 14:56:58,048 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsConfig", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-16 14:56:58,049 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-16 14:56:58,049 main DEBUG LoggerConfig$Builder(additivity="null", level="INFO", levelAndRefs="null", name="org.apache.hadoop.hbase.ScheduledChore", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-16 14:56:58,049 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-16 14:56:58,050 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.regionserver.RSRpcServices", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-16 14:56:58,050 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-16 14:56:58,050 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-16 14:56:58,051 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-16 14:56:58,051 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-16 14:56:58,051 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-16 14:56:58,052 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hbase.thirdparty.io.netty.channel", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-16 14:56:58,052 main DEBUG Building Plugin[name=root, class=org.apache.logging.log4j.core.config.LoggerConfig$RootLogger]. 2024-11-16 14:56:58,053 main DEBUG LoggerConfig$RootLogger$Builder(additivity="null", level="null", levelAndRefs="INFO,Console", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-16 14:56:58,055 main DEBUG Building Plugin[name=loggers, class=org.apache.logging.log4j.core.config.LoggersPlugin]. 2024-11-16 14:56:58,056 main DEBUG createLoggers(={org.apache.hadoop.metrics2.util.MBeans, org.apache.hadoop.hbase.logging.TestJul2Slf4j, org.apache.zookeeper, org.apache.hadoop.metrics2.impl.MetricsSinkAdapter, org.apache.hadoop.metrics2.impl.MetricsSystemImpl, org.apache.directory, org.apache.hadoop.hbase.ipc.FailedServers, org.apache.hadoop.metrics2.impl.MetricsConfig, org.apache.hadoop.hbase.ScheduledChore, org.apache.hadoop.hbase.regionserver.RSRpcServices, org.apache.hadoop, org.apache.hadoop.hbase, org.apache.hbase.thirdparty.io.netty.channel, root}) 2024-11-16 14:56:58,057 main DEBUG Building Plugin[name=layout, class=org.apache.logging.log4j.core.layout.PatternLayout]. 2024-11-16 14:56:58,058 main DEBUG PatternLayout$Builder(pattern="%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n", PatternSelector=null, Configuration(PropertiesConfig), Replace=null, charset="null", alwaysWriteExceptions="null", disableAnsi="null", noConsoleNoAnsi="null", header="null", footer="null") 2024-11-16 14:56:58,058 main DEBUG PluginManager 'Converter' found 47 plugins 2024-11-16 14:56:58,066 main DEBUG Building Plugin[name=appender, class=org.apache.hadoop.hbase.logging.HBaseTestAppender]. 2024-11-16 14:56:58,068 main DEBUG HBaseTestAppender$Builder(target="SYSTEM_ERR", maxSize="1G", bufferedIo="null", bufferSize="null", immediateFlush="null", ignoreExceptions="null", PatternLayout(%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n), name="Console", Configuration(PropertiesConfig), Filter=null, ={}) 2024-11-16 14:56:58,070 main DEBUG Starting HBaseTestOutputStreamManager SYSTEM_ERR 2024-11-16 14:56:58,070 main DEBUG Building Plugin[name=appenders, class=org.apache.logging.log4j.core.config.AppendersPlugin]. 2024-11-16 14:56:58,070 main DEBUG createAppenders(={Console}) 2024-11-16 14:56:58,071 main DEBUG Configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@15a34df2 initialized 2024-11-16 14:56:58,071 main DEBUG Starting configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@15a34df2 2024-11-16 14:56:58,072 main DEBUG Started configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@15a34df2 OK. 2024-11-16 14:56:58,072 main DEBUG Shutting down OutputStreamManager SYSTEM_OUT.false.false-1 2024-11-16 14:56:58,072 main DEBUG OutputStream closed 2024-11-16 14:56:58,072 main DEBUG Shut down OutputStreamManager SYSTEM_OUT.false.false-1, all resources released: true 2024-11-16 14:56:58,073 main DEBUG Appender DefaultConsole-1 stopped with status true 2024-11-16 14:56:58,073 main DEBUG Stopped org.apache.logging.log4j.core.config.DefaultConfiguration@4f2b503c OK 2024-11-16 14:56:58,141 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6 2024-11-16 14:56:58,142 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=StatusLogger 2024-11-16 14:56:58,143 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=ContextSelector 2024-11-16 14:56:58,144 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name= 2024-11-16 14:56:58,145 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.directory 2024-11-16 14:56:58,145 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSinkAdapter 2024-11-16 14:56:58,145 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.zookeeper 2024-11-16 14:56:58,145 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.logging.TestJul2Slf4j 2024-11-16 14:56:58,146 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSystemImpl 2024-11-16 14:56:58,146 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.util.MBeans 2024-11-16 14:56:58,146 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase 2024-11-16 14:56:58,147 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop 2024-11-16 14:56:58,147 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ipc.FailedServers 2024-11-16 14:56:58,147 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.regionserver.RSRpcServices 2024-11-16 14:56:58,148 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsConfig 2024-11-16 14:56:58,148 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hbase.thirdparty.io.netty.channel 2024-11-16 14:56:58,148 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ScheduledChore 2024-11-16 14:56:58,149 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Appenders,name=Console 2024-11-16 14:56:58,151 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-16 14:56:58,151 main DEBUG Reconfiguration complete for context[name=1dbd16a6] at URI jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-logging/target/hbase-logging-4.0.0-alpha-1-SNAPSHOT-tests.jar!/log4j2.properties (org.apache.logging.log4j.core.LoggerContext@31c7528f) with optional ClassLoader: null 2024-11-16 14:56:58,151 main DEBUG Shutdown hook enabled. Registering a new one. 2024-11-16 14:56:58,152 main DEBUG LoggerContext[name=1dbd16a6, org.apache.logging.log4j.core.LoggerContext@31c7528f] started OK. 2024-11-16T14:56:58,165 INFO [main {}] hbase.HBaseClassTestRule(94): Test class org.apache.hadoop.hbase.master.procedure.TestSCPWithMeta timeout: 13 mins 2024-11-16 14:56:58,167 main DEBUG AsyncLogger.ThreadNameStrategy=UNCACHED (user specified null, default is UNCACHED) 2024-11-16 14:56:58,168 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-16T14:56:58,203 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: master.procedure.TestSCPWithMeta#testRecoveryAndDoubleExecutionOnRsWithMeta Thread=11, OpenFileDescriptor=285, MaxFileDescriptor=1048576, SystemLoadAverage=82, ProcessCount=11, AvailableMemoryMB=6681 2024-11-16T14:56:58,474 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(323): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2c526703-9314-50ef-2cbc-7a77399c79d8 2024-11-16T14:56:58,475 INFO [Time-limited test {}] hbase.HBaseTestingUtil(805): Starting up minicluster with option: StartMiniClusterOption{numMasters=1, masterClass=null, numRegionServers=3, rsPorts=, rsClass=null, numDataNodes=3, dataNodeHosts=null, numZkServers=1, createRootDir=false, createWALDir=false} 2024-11-16T14:56:58,497 INFO [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2c526703-9314-50ef-2cbc-7a77399c79d8/cluster_74004074-8b3a-1fc1-ed3b-d8c79605d2dd, deleteOnExit=true 2024-11-16T14:56:58,498 INFO [Time-limited test {}] hbase.HBaseTestingUtil(818): STARTING DFS 2024-11-16T14:56:58,499 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2c526703-9314-50ef-2cbc-7a77399c79d8/test.cache.data in system properties and HBase conf 2024-11-16T14:56:58,500 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2c526703-9314-50ef-2cbc-7a77399c79d8/hadoop.tmp.dir in system properties and HBase conf 2024-11-16T14:56:58,501 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2c526703-9314-50ef-2cbc-7a77399c79d8/hadoop.log.dir in system properties and HBase conf 2024-11-16T14:56:58,502 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2c526703-9314-50ef-2cbc-7a77399c79d8/mapreduce.cluster.local.dir in system properties and HBase conf 2024-11-16T14:56:58,503 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2c526703-9314-50ef-2cbc-7a77399c79d8/mapreduce.cluster.temp.dir in system properties and HBase conf 2024-11-16T14:56:58,503 INFO [Time-limited test {}] hbase.HBaseTestingUtil(738): read short circuit is OFF 2024-11-16T14:56:58,612 WARN [Time-limited test {}] util.NativeCodeLoader(60): Unable to load native-hadoop library for your platform... using builtin-java classes where applicable 2024-11-16T14:56:58,735 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-16T14:56:58,740 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2c526703-9314-50ef-2cbc-7a77399c79d8/yarn.node-labels.fs-store.root-dir in system properties and HBase conf 2024-11-16T14:56:58,741 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2c526703-9314-50ef-2cbc-7a77399c79d8/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf 2024-11-16T14:56:58,741 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2c526703-9314-50ef-2cbc-7a77399c79d8/yarn.nodemanager.log-dirs in system properties and HBase conf 2024-11-16T14:56:58,742 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2c526703-9314-50ef-2cbc-7a77399c79d8/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-16T14:56:58,743 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2c526703-9314-50ef-2cbc-7a77399c79d8/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf 2024-11-16T14:56:58,743 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2c526703-9314-50ef-2cbc-7a77399c79d8/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf 2024-11-16T14:56:58,744 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2c526703-9314-50ef-2cbc-7a77399c79d8/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-16T14:56:58,744 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2c526703-9314-50ef-2cbc-7a77399c79d8/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-16T14:56:58,745 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2c526703-9314-50ef-2cbc-7a77399c79d8/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf 2024-11-16T14:56:58,745 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2c526703-9314-50ef-2cbc-7a77399c79d8/nfs.dump.dir in system properties and HBase conf 2024-11-16T14:56:58,746 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2c526703-9314-50ef-2cbc-7a77399c79d8/java.io.tmpdir in system properties and HBase conf 2024-11-16T14:56:58,746 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2c526703-9314-50ef-2cbc-7a77399c79d8/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-16T14:56:58,747 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2c526703-9314-50ef-2cbc-7a77399c79d8/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf 2024-11-16T14:56:58,748 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2c526703-9314-50ef-2cbc-7a77399c79d8/fs.s3a.committer.staging.tmp.path in system properties and HBase conf 2024-11-16T14:56:59,699 WARN [Time-limited test {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-namenode.properties,hadoop-metrics2.properties 2024-11-16T14:56:59,771 INFO [Time-limited test {}] log.Log(170): Logging initialized @2393ms to org.eclipse.jetty.util.log.Slf4jLog 2024-11-16T14:56:59,838 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-16T14:56:59,900 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-16T14:56:59,919 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-16T14:56:59,919 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-16T14:56:59,921 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-11-16T14:56:59,932 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-16T14:56:59,935 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@58897554{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2c526703-9314-50ef-2cbc-7a77399c79d8/hadoop.log.dir/,AVAILABLE} 2024-11-16T14:56:59,936 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@2a372edd{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-16T14:57:00,109 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@1aa62678{hdfs,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2c526703-9314-50ef-2cbc-7a77399c79d8/java.io.tmpdir/jetty-localhost-42265-hadoop-hdfs-3_4_1-tests_jar-_-any-12591509996831325585/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-11-16T14:57:00,116 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@1c0c42ad{HTTP/1.1, (http/1.1)}{localhost:42265} 2024-11-16T14:57:00,117 INFO [Time-limited test {}] server.Server(415): Started @2738ms 2024-11-16T14:57:00,605 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-16T14:57:00,612 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-16T14:57:00,614 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-16T14:57:00,614 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-16T14:57:00,614 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-11-16T14:57:00,615 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@b71ffd0{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2c526703-9314-50ef-2cbc-7a77399c79d8/hadoop.log.dir/,AVAILABLE} 2024-11-16T14:57:00,615 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@63d82da9{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-16T14:57:00,712 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@4255463d{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2c526703-9314-50ef-2cbc-7a77399c79d8/java.io.tmpdir/jetty-localhost-40101-hadoop-hdfs-3_4_1-tests_jar-_-any-16934609585645693100/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-16T14:57:00,712 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@5e48d68a{HTTP/1.1, (http/1.1)}{localhost:40101} 2024-11-16T14:57:00,713 INFO [Time-limited test {}] server.Server(415): Started @3335ms 2024-11-16T14:57:00,760 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-11-16T14:57:00,865 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-16T14:57:00,872 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-16T14:57:00,874 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-16T14:57:00,874 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-16T14:57:00,874 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-11-16T14:57:00,875 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@63779849{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2c526703-9314-50ef-2cbc-7a77399c79d8/hadoop.log.dir/,AVAILABLE} 2024-11-16T14:57:00,876 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@a292a59{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-16T14:57:00,982 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@3635c31e{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2c526703-9314-50ef-2cbc-7a77399c79d8/java.io.tmpdir/jetty-localhost-43803-hadoop-hdfs-3_4_1-tests_jar-_-any-3052525947242784100/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-16T14:57:00,982 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@78839a64{HTTP/1.1, (http/1.1)}{localhost:43803} 2024-11-16T14:57:00,983 INFO [Time-limited test {}] server.Server(415): Started @3605ms 2024-11-16T14:57:00,985 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-11-16T14:57:01,023 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-16T14:57:01,028 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-16T14:57:01,030 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-16T14:57:01,030 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-16T14:57:01,030 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-11-16T14:57:01,032 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@7ed2891b{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2c526703-9314-50ef-2cbc-7a77399c79d8/hadoop.log.dir/,AVAILABLE} 2024-11-16T14:57:01,033 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@43bf9dc1{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-16T14:57:01,146 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@486c2677{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2c526703-9314-50ef-2cbc-7a77399c79d8/java.io.tmpdir/jetty-localhost-44051-hadoop-hdfs-3_4_1-tests_jar-_-any-1680883326862124478/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-16T14:57:01,147 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@6738d157{HTTP/1.1, (http/1.1)}{localhost:44051} 2024-11-16T14:57:01,147 INFO [Time-limited test {}] server.Server(415): Started @3769ms 2024-11-16T14:57:01,150 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-11-16T14:57:01,951 WARN [Thread-123 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2c526703-9314-50ef-2cbc-7a77399c79d8/cluster_74004074-8b3a-1fc1-ed3b-d8c79605d2dd/data/data4/current/BP-461342148-172.17.0.2-1731769019248/current, will proceed with Du for space computation calculation, 2024-11-16T14:57:01,951 WARN [Thread-125 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2c526703-9314-50ef-2cbc-7a77399c79d8/cluster_74004074-8b3a-1fc1-ed3b-d8c79605d2dd/data/data2/current/BP-461342148-172.17.0.2-1731769019248/current, will proceed with Du for space computation calculation, 2024-11-16T14:57:01,951 WARN [Thread-124 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2c526703-9314-50ef-2cbc-7a77399c79d8/cluster_74004074-8b3a-1fc1-ed3b-d8c79605d2dd/data/data1/current/BP-461342148-172.17.0.2-1731769019248/current, will proceed with Du for space computation calculation, 2024-11-16T14:57:01,951 WARN [Thread-122 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2c526703-9314-50ef-2cbc-7a77399c79d8/cluster_74004074-8b3a-1fc1-ed3b-d8c79605d2dd/data/data3/current/BP-461342148-172.17.0.2-1731769019248/current, will proceed with Du for space computation calculation, 2024-11-16T14:57:01,984 WARN [Thread-58 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-11-16T14:57:01,984 WARN [Thread-81 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-11-16T14:57:02,028 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x2f096bcae9080e45 with lease ID 0xb8377b0d0163fd84: Processing first storage report for DS-69dfd3e9-4ae1-4c6e-b575-41b55f386dcc from datanode DatanodeRegistration(127.0.0.1:36531, datanodeUuid=3277bc61-d18b-46f8-bf1f-cc24ec81b9e9, infoPort=39249, infoSecurePort=0, ipcPort=44373, storageInfo=lv=-57;cid=testClusterID;nsid=860561919;c=1731769019248) 2024-11-16T14:57:02,029 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x2f096bcae9080e45 with lease ID 0xb8377b0d0163fd84: from storage DS-69dfd3e9-4ae1-4c6e-b575-41b55f386dcc node DatanodeRegistration(127.0.0.1:36531, datanodeUuid=3277bc61-d18b-46f8-bf1f-cc24ec81b9e9, infoPort=39249, infoSecurePort=0, ipcPort=44373, storageInfo=lv=-57;cid=testClusterID;nsid=860561919;c=1731769019248), blocks: 0, hasStaleStorage: true, processing time: 1 msecs, invalidatedBlocks: 0 2024-11-16T14:57:02,030 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x8dd33766c3b32251 with lease ID 0xb8377b0d0163fd83: Processing first storage report for DS-cfbd562b-a0b9-4a9e-8390-a5133b0d2c73 from datanode DatanodeRegistration(127.0.0.1:41671, datanodeUuid=7b37a58c-887c-4bbb-9985-3e291a925997, infoPort=40829, infoSecurePort=0, ipcPort=33723, storageInfo=lv=-57;cid=testClusterID;nsid=860561919;c=1731769019248) 2024-11-16T14:57:02,030 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x8dd33766c3b32251 with lease ID 0xb8377b0d0163fd83: from storage DS-cfbd562b-a0b9-4a9e-8390-a5133b0d2c73 node DatanodeRegistration(127.0.0.1:41671, datanodeUuid=7b37a58c-887c-4bbb-9985-3e291a925997, infoPort=40829, infoSecurePort=0, ipcPort=33723, storageInfo=lv=-57;cid=testClusterID;nsid=860561919;c=1731769019248), blocks: 0, hasStaleStorage: true, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-16T14:57:02,030 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x2f096bcae9080e45 with lease ID 0xb8377b0d0163fd84: Processing first storage report for DS-8319c95a-887a-4e81-9e96-2f989a3963b0 from datanode DatanodeRegistration(127.0.0.1:36531, datanodeUuid=3277bc61-d18b-46f8-bf1f-cc24ec81b9e9, infoPort=39249, infoSecurePort=0, ipcPort=44373, storageInfo=lv=-57;cid=testClusterID;nsid=860561919;c=1731769019248) 2024-11-16T14:57:02,030 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x2f096bcae9080e45 with lease ID 0xb8377b0d0163fd84: from storage DS-8319c95a-887a-4e81-9e96-2f989a3963b0 node DatanodeRegistration(127.0.0.1:36531, datanodeUuid=3277bc61-d18b-46f8-bf1f-cc24ec81b9e9, infoPort=39249, infoSecurePort=0, ipcPort=44373, storageInfo=lv=-57;cid=testClusterID;nsid=860561919;c=1731769019248), blocks: 0, hasStaleStorage: false, processing time: 1 msecs, invalidatedBlocks: 0 2024-11-16T14:57:02,031 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x8dd33766c3b32251 with lease ID 0xb8377b0d0163fd83: Processing first storage report for DS-921494ae-2a74-4ca9-a8ef-4c7ef088c1c1 from datanode DatanodeRegistration(127.0.0.1:41671, datanodeUuid=7b37a58c-887c-4bbb-9985-3e291a925997, infoPort=40829, infoSecurePort=0, ipcPort=33723, storageInfo=lv=-57;cid=testClusterID;nsid=860561919;c=1731769019248) 2024-11-16T14:57:02,031 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x8dd33766c3b32251 with lease ID 0xb8377b0d0163fd83: from storage DS-921494ae-2a74-4ca9-a8ef-4c7ef088c1c1 node DatanodeRegistration(127.0.0.1:41671, datanodeUuid=7b37a58c-887c-4bbb-9985-3e291a925997, infoPort=40829, infoSecurePort=0, ipcPort=33723, storageInfo=lv=-57;cid=testClusterID;nsid=860561919;c=1731769019248), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-16T14:57:02,061 WARN [Thread-143 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2c526703-9314-50ef-2cbc-7a77399c79d8/cluster_74004074-8b3a-1fc1-ed3b-d8c79605d2dd/data/data6/current/BP-461342148-172.17.0.2-1731769019248/current, will proceed with Du for space computation calculation, 2024-11-16T14:57:02,061 WARN [Thread-142 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2c526703-9314-50ef-2cbc-7a77399c79d8/cluster_74004074-8b3a-1fc1-ed3b-d8c79605d2dd/data/data5/current/BP-461342148-172.17.0.2-1731769019248/current, will proceed with Du for space computation calculation, 2024-11-16T14:57:02,078 WARN [Thread-103 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-11-16T14:57:02,082 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x5088bbdbc59b39d1 with lease ID 0xb8377b0d0163fd85: Processing first storage report for DS-cecddcef-3bfd-4cf3-a998-7e060881adbc from datanode DatanodeRegistration(127.0.0.1:40641, datanodeUuid=8a23e5b4-97ec-4720-90a3-a377bbd0a053, infoPort=35235, infoSecurePort=0, ipcPort=46007, storageInfo=lv=-57;cid=testClusterID;nsid=860561919;c=1731769019248) 2024-11-16T14:57:02,082 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x5088bbdbc59b39d1 with lease ID 0xb8377b0d0163fd85: from storage DS-cecddcef-3bfd-4cf3-a998-7e060881adbc node DatanodeRegistration(127.0.0.1:40641, datanodeUuid=8a23e5b4-97ec-4720-90a3-a377bbd0a053, infoPort=35235, infoSecurePort=0, ipcPort=46007, storageInfo=lv=-57;cid=testClusterID;nsid=860561919;c=1731769019248), blocks: 0, hasStaleStorage: true, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-16T14:57:02,082 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x5088bbdbc59b39d1 with lease ID 0xb8377b0d0163fd85: Processing first storage report for DS-838a2997-1b0b-4482-9b03-1fa3a8923503 from datanode DatanodeRegistration(127.0.0.1:40641, datanodeUuid=8a23e5b4-97ec-4720-90a3-a377bbd0a053, infoPort=35235, infoSecurePort=0, ipcPort=46007, storageInfo=lv=-57;cid=testClusterID;nsid=860561919;c=1731769019248) 2024-11-16T14:57:02,082 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x5088bbdbc59b39d1 with lease ID 0xb8377b0d0163fd85: from storage DS-838a2997-1b0b-4482-9b03-1fa3a8923503 node DatanodeRegistration(127.0.0.1:40641, datanodeUuid=8a23e5b4-97ec-4720-90a3-a377bbd0a053, infoPort=35235, infoSecurePort=0, ipcPort=46007, storageInfo=lv=-57;cid=testClusterID;nsid=860561919;c=1731769019248), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-16T14:57:02,167 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(631): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2c526703-9314-50ef-2cbc-7a77399c79d8 2024-11-16T14:57:02,225 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2c526703-9314-50ef-2cbc-7a77399c79d8/cluster_74004074-8b3a-1fc1-ed3b-d8c79605d2dd/zookeeper_0, clientPort=63224, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2c526703-9314-50ef-2cbc-7a77399c79d8/cluster_74004074-8b3a-1fc1-ed3b-d8c79605d2dd/zookeeper_0/version-2, dataDirSize=457 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2c526703-9314-50ef-2cbc-7a77399c79d8/cluster_74004074-8b3a-1fc1-ed3b-d8c79605d2dd/zookeeper_0/version-2, dataLogSize=457 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0 2024-11-16T14:57:02,234 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=63224 2024-11-16T14:57:02,243 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-16T14:57:02,246 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-16T14:57:02,447 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36531 is added to blk_1073741825_1001 (size=7) 2024-11-16T14:57:02,448 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40641 is added to blk_1073741825_1001 (size=7) 2024-11-16T14:57:02,448 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41671 is added to blk_1073741825_1001 (size=7) 2024-11-16T14:57:02,863 INFO [Time-limited test {}] util.FSUtils(489): Created version file at hdfs://localhost:42223/user/jenkins/test-data/a8176cb1-2ebb-b530-b3c7-53c9e53c186e with version=8 2024-11-16T14:57:02,863 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1139): Setting hbase.fs.tmp.dir to hdfs://localhost:42223/user/jenkins/test-data/a8176cb1-2ebb-b530-b3c7-53c9e53c186e/hbase-staging 2024-11-16T14:57:02,941 DEBUG [Time-limited test {}] channel.MultithreadEventLoopGroup(44): -Dio.netty.eventLoopThreads: 16 2024-11-16T14:57:03,194 INFO [Time-limited test {}] client.ConnectionUtils(128): master/fc3fda29f660:0 server-side Connection retries=45 2024-11-16T14:57:03,202 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-16T14:57:03,202 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-16T14:57:03,207 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-16T14:57:03,207 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-16T14:57:03,207 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-16T14:57:03,333 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService 2024-11-16T14:57:03,383 INFO [Time-limited test {}] metrics.MetricRegistriesLoader(60): Loaded MetricRegistries class org.apache.hadoop.hbase.metrics.impl.MetricRegistriesImpl 2024-11-16T14:57:03,391 DEBUG [Time-limited test {}] util.ClassSize(228): Using Unsafe to estimate memory layout 2024-11-16T14:57:03,394 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-16T14:57:03,416 DEBUG [Time-limited test {}] channel.DefaultChannelId(84): -Dio.netty.processId: 99014 (auto-detected) 2024-11-16T14:57:03,417 DEBUG [Time-limited test {}] channel.DefaultChannelId(106): -Dio.netty.machineId: 02:42:ac:ff:fe:11:00:02 (auto-detected) 2024-11-16T14:57:03,435 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:43615 2024-11-16T14:57:03,454 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:43615 connecting to ZooKeeper ensemble=127.0.0.1:63224 2024-11-16T14:57:03,580 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:436150x0, quorum=127.0.0.1:63224, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-16T14:57:03,582 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:43615-0x101456fc9c70000 connected 2024-11-16T14:57:03,656 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-16T14:57:03,663 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-16T14:57:03,673 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:43615-0x101456fc9c70000, quorum=127.0.0.1:63224, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-16T14:57:03,677 INFO [Time-limited test {}] master.HMaster(525): hbase.rootdir=hdfs://localhost:42223/user/jenkins/test-data/a8176cb1-2ebb-b530-b3c7-53c9e53c186e, hbase.cluster.distributed=false 2024-11-16T14:57:03,701 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:43615-0x101456fc9c70000, quorum=127.0.0.1:63224, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-16T14:57:03,706 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=43615 2024-11-16T14:57:03,706 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=43615 2024-11-16T14:57:03,707 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=43615 2024-11-16T14:57:03,708 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=43615 2024-11-16T14:57:03,709 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=43615 2024-11-16T14:57:03,800 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/fc3fda29f660:0 server-side Connection retries=45 2024-11-16T14:57:03,802 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-16T14:57:03,802 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-16T14:57:03,802 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-16T14:57:03,802 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-16T14:57:03,802 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-16T14:57:03,805 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-16T14:57:03,807 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-16T14:57:03,808 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:39089 2024-11-16T14:57:03,810 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:39089 connecting to ZooKeeper ensemble=127.0.0.1:63224 2024-11-16T14:57:03,811 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-16T14:57:03,813 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-16T14:57:03,837 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:390890x0, quorum=127.0.0.1:63224, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-16T14:57:03,838 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:39089-0x101456fc9c70001 connected 2024-11-16T14:57:03,838 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:39089-0x101456fc9c70001, quorum=127.0.0.1:63224, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-16T14:57:03,844 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-16T14:57:03,852 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-16T14:57:03,854 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:39089-0x101456fc9c70001, quorum=127.0.0.1:63224, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-16T14:57:03,859 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:39089-0x101456fc9c70001, quorum=127.0.0.1:63224, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-16T14:57:03,860 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=39089 2024-11-16T14:57:03,861 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=39089 2024-11-16T14:57:03,861 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=39089 2024-11-16T14:57:03,862 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=39089 2024-11-16T14:57:03,862 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=39089 2024-11-16T14:57:03,879 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/fc3fda29f660:0 server-side Connection retries=45 2024-11-16T14:57:03,879 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-16T14:57:03,880 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-16T14:57:03,880 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-16T14:57:03,880 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-16T14:57:03,880 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-16T14:57:03,880 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-16T14:57:03,881 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-16T14:57:03,882 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:38897 2024-11-16T14:57:03,884 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:38897 connecting to ZooKeeper ensemble=127.0.0.1:63224 2024-11-16T14:57:03,885 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-16T14:57:03,887 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-16T14:57:03,921 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:388970x0, quorum=127.0.0.1:63224, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-16T14:57:03,922 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:388970x0, quorum=127.0.0.1:63224, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-16T14:57:03,922 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:38897-0x101456fc9c70002 connected 2024-11-16T14:57:03,922 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-16T14:57:03,924 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-16T14:57:03,925 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:38897-0x101456fc9c70002, quorum=127.0.0.1:63224, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-16T14:57:03,928 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:38897-0x101456fc9c70002, quorum=127.0.0.1:63224, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-16T14:57:03,929 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=38897 2024-11-16T14:57:03,930 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=38897 2024-11-16T14:57:03,931 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=38897 2024-11-16T14:57:03,932 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=38897 2024-11-16T14:57:03,932 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=38897 2024-11-16T14:57:03,951 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/fc3fda29f660:0 server-side Connection retries=45 2024-11-16T14:57:03,952 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-16T14:57:03,952 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-16T14:57:03,952 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-16T14:57:03,952 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-16T14:57:03,952 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-16T14:57:03,953 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-16T14:57:03,953 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-16T14:57:03,954 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:42985 2024-11-16T14:57:03,956 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:42985 connecting to ZooKeeper ensemble=127.0.0.1:63224 2024-11-16T14:57:03,957 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-16T14:57:03,959 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-16T14:57:03,970 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:429850x0, quorum=127.0.0.1:63224, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-16T14:57:03,971 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:429850x0, quorum=127.0.0.1:63224, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-16T14:57:03,971 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:42985-0x101456fc9c70003 connected 2024-11-16T14:57:03,972 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-16T14:57:03,973 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-16T14:57:03,974 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:42985-0x101456fc9c70003, quorum=127.0.0.1:63224, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-16T14:57:03,976 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:42985-0x101456fc9c70003, quorum=127.0.0.1:63224, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-16T14:57:03,977 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=42985 2024-11-16T14:57:03,978 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=42985 2024-11-16T14:57:03,978 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=42985 2024-11-16T14:57:03,980 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=42985 2024-11-16T14:57:03,980 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=42985 2024-11-16T14:57:03,994 DEBUG [M:0;fc3fda29f660:43615 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;fc3fda29f660:43615 2024-11-16T14:57:03,996 INFO [master/fc3fda29f660:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/fc3fda29f660,43615,1731769023046 2024-11-16T14:57:04,012 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42985-0x101456fc9c70003, quorum=127.0.0.1:63224, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-16T14:57:04,012 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43615-0x101456fc9c70000, quorum=127.0.0.1:63224, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-16T14:57:04,012 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39089-0x101456fc9c70001, quorum=127.0.0.1:63224, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-16T14:57:04,012 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38897-0x101456fc9c70002, quorum=127.0.0.1:63224, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-16T14:57:04,021 DEBUG [master/fc3fda29f660:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:43615-0x101456fc9c70000, quorum=127.0.0.1:63224, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/fc3fda29f660,43615,1731769023046 2024-11-16T14:57:04,045 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43615-0x101456fc9c70000, quorum=127.0.0.1:63224, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-16T14:57:04,045 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42985-0x101456fc9c70003, quorum=127.0.0.1:63224, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-16T14:57:04,045 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39089-0x101456fc9c70001, quorum=127.0.0.1:63224, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-16T14:57:04,045 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38897-0x101456fc9c70002, quorum=127.0.0.1:63224, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-16T14:57:04,045 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42985-0x101456fc9c70003, quorum=127.0.0.1:63224, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-16T14:57:04,045 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39089-0x101456fc9c70001, quorum=127.0.0.1:63224, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-16T14:57:04,045 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38897-0x101456fc9c70002, quorum=127.0.0.1:63224, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-16T14:57:04,046 DEBUG [master/fc3fda29f660:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:43615-0x101456fc9c70000, quorum=127.0.0.1:63224, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-11-16T14:57:04,047 INFO [master/fc3fda29f660:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/fc3fda29f660,43615,1731769023046 from backup master directory 2024-11-16T14:57:04,053 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43615-0x101456fc9c70000, quorum=127.0.0.1:63224, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/fc3fda29f660,43615,1731769023046 2024-11-16T14:57:04,053 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39089-0x101456fc9c70001, quorum=127.0.0.1:63224, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-16T14:57:04,053 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42985-0x101456fc9c70003, quorum=127.0.0.1:63224, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-16T14:57:04,053 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38897-0x101456fc9c70002, quorum=127.0.0.1:63224, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-16T14:57:04,053 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43615-0x101456fc9c70000, quorum=127.0.0.1:63224, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-16T14:57:04,054 WARN [master/fc3fda29f660:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-16T14:57:04,055 INFO [master/fc3fda29f660:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=fc3fda29f660,43615,1731769023046 2024-11-16T14:57:04,057 INFO [master/fc3fda29f660:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating data MemStoreChunkPool with chunk size 2 MB, max count 396, initial count 0 2024-11-16T14:57:04,059 INFO [master/fc3fda29f660:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating index MemStoreChunkPool with chunk size 204.80 KB, max count 440, initial count 0 2024-11-16T14:57:04,112 DEBUG [master/fc3fda29f660:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [hdfs://localhost:42223/user/jenkins/test-data/a8176cb1-2ebb-b530-b3c7-53c9e53c186e/hbase.id] with ID: a2ba0a3e-7753-4646-b6b2-e5ea1356a3cb 2024-11-16T14:57:04,112 DEBUG [master/fc3fda29f660:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: hdfs://localhost:42223/user/jenkins/test-data/a8176cb1-2ebb-b530-b3c7-53c9e53c186e/.tmp/hbase.id 2024-11-16T14:57:04,127 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40641 is added to blk_1073741826_1002 (size=42) 2024-11-16T14:57:04,127 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41671 is added to blk_1073741826_1002 (size=42) 2024-11-16T14:57:04,128 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36531 is added to blk_1073741826_1002 (size=42) 2024-11-16T14:57:04,129 DEBUG [master/fc3fda29f660:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [hdfs://localhost:42223/user/jenkins/test-data/a8176cb1-2ebb-b530-b3c7-53c9e53c186e/.tmp/hbase.id]:[hdfs://localhost:42223/user/jenkins/test-data/a8176cb1-2ebb-b530-b3c7-53c9e53c186e/hbase.id] 2024-11-16T14:57:04,178 INFO [master/fc3fda29f660:0:becomeActiveMaster {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-16T14:57:04,182 INFO [master/fc3fda29f660:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem. 2024-11-16T14:57:04,201 INFO [master/fc3fda29f660:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 18ms. 2024-11-16T14:57:04,229 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38897-0x101456fc9c70002, quorum=127.0.0.1:63224, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-16T14:57:04,229 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42985-0x101456fc9c70003, quorum=127.0.0.1:63224, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-16T14:57:04,229 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39089-0x101456fc9c70001, quorum=127.0.0.1:63224, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-16T14:57:04,229 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43615-0x101456fc9c70000, quorum=127.0.0.1:63224, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-16T14:57:04,249 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36531 is added to blk_1073741827_1003 (size=196) 2024-11-16T14:57:04,249 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41671 is added to blk_1073741827_1003 (size=196) 2024-11-16T14:57:04,250 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40641 is added to blk_1073741827_1003 (size=196) 2024-11-16T14:57:04,266 INFO [master/fc3fda29f660:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-16T14:57:04,269 INFO [master/fc3fda29f660:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-11-16T14:57:04,283 DEBUG [master/fc3fda29f660:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(244): No decryptEncryptedDataEncryptionKey method in DFSClient, should be hadoop version with HDFS-12396 java.lang.NoSuchMethodException: org.apache.hadoop.hdfs.DFSClient.decryptEncryptedDataEncryptionKey(org.apache.hadoop.fs.FileEncryptionInfo) at java.lang.Class.getDeclaredMethod(Class.java:2675) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelperWithoutHDFS12396(FanOutOneBlockAsyncDFSOutputSaslHelper.java:183) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelper(FanOutOneBlockAsyncDFSOutputSaslHelper.java:242) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.(FanOutOneBlockAsyncDFSOutputSaslHelper.java:253) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.lang.Class.forName0(Native Method) ~[?:?] at java.lang.Class.forName(Class.java:375) ~[?:?] at org.apache.hadoop.hbase.wal.AsyncFSWALProvider.load(AsyncFSWALProvider.java:150) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.getProviderClass(WALFactory.java:174) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:262) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:231) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegion.create(MasterRegion.java:400) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegionFactory.create(MasterRegionFactory.java:135) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.finishActiveMasterInitialization(HMaster.java:1003) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.startActiveMasterManager(HMaster.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.lambda$run$0(HMaster.java:613) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.lambda$tracedRunnable$2(TraceUtil.java:155) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:57:04,286 INFO [master/fc3fda29f660:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-16T14:57:04,317 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36531 is added to blk_1073741828_1004 (size=1189) 2024-11-16T14:57:04,317 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40641 is added to blk_1073741828_1004 (size=1189) 2024-11-16T14:57:04,318 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41671 is added to blk_1073741828_1004 (size=1189) 2024-11-16T14:57:04,336 INFO [master/fc3fda29f660:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir hdfs://localhost:42223/user/jenkins/test-data/a8176cb1-2ebb-b530-b3c7-53c9e53c186e/MasterData/data/master/store 2024-11-16T14:57:04,356 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41671 is added to blk_1073741829_1005 (size=34) 2024-11-16T14:57:04,356 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40641 is added to blk_1073741829_1005 (size=34) 2024-11-16T14:57:04,356 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36531 is added to blk_1073741829_1005 (size=34) 2024-11-16T14:57:04,361 INFO [master/fc3fda29f660:0:becomeActiveMaster {}] throttle.StoreHotnessProtector(112): StoreHotnessProtector is disabled. Set hbase.region.store.parallel.put.limit > 0 to enable, which may help mitigate load under heavy write pressure. 2024-11-16T14:57:04,365 DEBUG [master/fc3fda29f660:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-16T14:57:04,366 DEBUG [master/fc3fda29f660:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-16T14:57:04,366 INFO [master/fc3fda29f660:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-16T14:57:04,366 DEBUG [master/fc3fda29f660:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-16T14:57:04,368 DEBUG [master/fc3fda29f660:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-16T14:57:04,368 DEBUG [master/fc3fda29f660:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-16T14:57:04,368 INFO [master/fc3fda29f660:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-16T14:57:04,369 DEBUG [master/fc3fda29f660:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1731769024366Disabling compacts and flushes for region at 1731769024366Disabling writes for close at 1731769024368 (+2 ms)Writing region close event to WAL at 1731769024368Closed at 1731769024368 2024-11-16T14:57:04,372 WARN [master/fc3fda29f660:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: hdfs://localhost:42223/user/jenkins/test-data/a8176cb1-2ebb-b530-b3c7-53c9e53c186e/MasterData/data/master/store/.initializing 2024-11-16T14:57:04,372 DEBUG [master/fc3fda29f660:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=hdfs://localhost:42223/user/jenkins/test-data/a8176cb1-2ebb-b530-b3c7-53c9e53c186e/MasterData/WALs/fc3fda29f660,43615,1731769023046 2024-11-16T14:57:04,380 INFO [master/fc3fda29f660:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-16T14:57:04,393 INFO [master/fc3fda29f660:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=fc3fda29f660%2C43615%2C1731769023046, suffix=, logDir=hdfs://localhost:42223/user/jenkins/test-data/a8176cb1-2ebb-b530-b3c7-53c9e53c186e/MasterData/WALs/fc3fda29f660,43615,1731769023046, archiveDir=hdfs://localhost:42223/user/jenkins/test-data/a8176cb1-2ebb-b530-b3c7-53c9e53c186e/MasterData/oldWALs, maxLogs=10 2024-11-16T14:57:04,415 DEBUG [master/fc3fda29f660:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/a8176cb1-2ebb-b530-b3c7-53c9e53c186e/MasterData/WALs/fc3fda29f660,43615,1731769023046/fc3fda29f660%2C43615%2C1731769023046.1731769024397, exclude list is [], retry=0 2024-11-16T14:57:04,430 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36531,DS-69dfd3e9-4ae1-4c6e-b575-41b55f386dcc,DISK] 2024-11-16T14:57:04,430 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41671,DS-cfbd562b-a0b9-4a9e-8390-a5133b0d2c73,DISK] 2024-11-16T14:57:04,430 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40641,DS-cecddcef-3bfd-4cf3-a998-7e060881adbc,DISK] 2024-11-16T14:57:04,433 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.ProtobufDecoder(117): Hadoop 3.3 and above shades protobuf. 2024-11-16T14:57:04,471 INFO [master/fc3fda29f660:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/a8176cb1-2ebb-b530-b3c7-53c9e53c186e/MasterData/WALs/fc3fda29f660,43615,1731769023046/fc3fda29f660%2C43615%2C1731769023046.1731769024397 2024-11-16T14:57:04,472 DEBUG [master/fc3fda29f660:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:40829:40829),(127.0.0.1/127.0.0.1:35235:35235),(127.0.0.1/127.0.0.1:39249:39249)] 2024-11-16T14:57:04,473 DEBUG [master/fc3fda29f660:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-11-16T14:57:04,473 DEBUG [master/fc3fda29f660:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-16T14:57:04,476 DEBUG [master/fc3fda29f660:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-11-16T14:57:04,476 DEBUG [master/fc3fda29f660:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-11-16T14:57:04,510 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-11-16T14:57:04,532 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-11-16T14:57:04,535 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:57:04,537 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-16T14:57:04,537 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-11-16T14:57:04,541 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-11-16T14:57:04,541 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:57:04,542 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-16T14:57:04,542 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-11-16T14:57:04,545 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-11-16T14:57:04,545 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:57:04,546 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-16T14:57:04,546 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-11-16T14:57:04,549 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-11-16T14:57:04,549 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:57:04,550 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-16T14:57:04,550 DEBUG [master/fc3fda29f660:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682 2024-11-16T14:57:04,554 DEBUG [master/fc3fda29f660:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42223/user/jenkins/test-data/a8176cb1-2ebb-b530-b3c7-53c9e53c186e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-16T14:57:04,555 DEBUG [master/fc3fda29f660:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42223/user/jenkins/test-data/a8176cb1-2ebb-b530-b3c7-53c9e53c186e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-16T14:57:04,560 DEBUG [master/fc3fda29f660:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682 2024-11-16T14:57:04,561 DEBUG [master/fc3fda29f660:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682 2024-11-16T14:57:04,565 DEBUG [master/fc3fda29f660:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-16T14:57:04,569 DEBUG [master/fc3fda29f660:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-11-16T14:57:04,573 DEBUG [master/fc3fda29f660:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:42223/user/jenkins/test-data/a8176cb1-2ebb-b530-b3c7-53c9e53c186e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-16T14:57:04,574 INFO [master/fc3fda29f660:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=64140149, jitterRate=-0.044237300753593445}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-16T14:57:04,581 DEBUG [master/fc3fda29f660:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682: Writing region info on filesystem at 1731769024487Initializing all the Stores at 1731769024489 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731769024490 (+1 ms)Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731769024491 (+1 ms)Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731769024491Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731769024491Cleaning up temporary data from old regions at 1731769024561 (+70 ms)Region opened successfully at 1731769024581 (+20 ms) 2024-11-16T14:57:04,582 INFO [master/fc3fda29f660:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-11-16T14:57:04,612 DEBUG [master/fc3fda29f660:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@31b9e85, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=fc3fda29f660/172.17.0.2:0 2024-11-16T14:57:04,638 INFO [master/fc3fda29f660:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating... 2024-11-16T14:57:04,648 INFO [master/fc3fda29f660:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-16T14:57:04,648 INFO [master/fc3fda29f660:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-16T14:57:04,651 INFO [master/fc3fda29f660:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-16T14:57:04,652 INFO [master/fc3fda29f660:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 1 msec 2024-11-16T14:57:04,657 INFO [master/fc3fda29f660:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 4 msec 2024-11-16T14:57:04,657 INFO [master/fc3fda29f660:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-11-16T14:57:04,682 INFO [master/fc3fda29f660:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2024-11-16T14:57:04,690 DEBUG [master/fc3fda29f660:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:43615-0x101456fc9c70000, quorum=127.0.0.1:63224, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error) 2024-11-16T14:57:04,728 DEBUG [master/fc3fda29f660:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false 2024-11-16T14:57:04,732 INFO [master/fc3fda29f660:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2024-11-16T14:57:04,735 DEBUG [master/fc3fda29f660:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:43615-0x101456fc9c70000, quorum=127.0.0.1:63224, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error) 2024-11-16T14:57:04,745 DEBUG [master/fc3fda29f660:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false 2024-11-16T14:57:04,747 INFO [master/fc3fda29f660:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2024-11-16T14:57:04,751 DEBUG [master/fc3fda29f660:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:43615-0x101456fc9c70000, quorum=127.0.0.1:63224, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error) 2024-11-16T14:57:04,761 DEBUG [master/fc3fda29f660:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false 2024-11-16T14:57:04,763 DEBUG [master/fc3fda29f660:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:43615-0x101456fc9c70000, quorum=127.0.0.1:63224, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error) 2024-11-16T14:57:04,769 DEBUG [master/fc3fda29f660:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false 2024-11-16T14:57:04,792 DEBUG [master/fc3fda29f660:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:43615-0x101456fc9c70000, quorum=127.0.0.1:63224, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error) 2024-11-16T14:57:04,803 DEBUG [master/fc3fda29f660:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false 2024-11-16T14:57:04,811 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43615-0x101456fc9c70000, quorum=127.0.0.1:63224, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-16T14:57:04,811 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42985-0x101456fc9c70003, quorum=127.0.0.1:63224, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-16T14:57:04,812 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43615-0x101456fc9c70000, quorum=127.0.0.1:63224, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-16T14:57:04,812 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42985-0x101456fc9c70003, quorum=127.0.0.1:63224, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-16T14:57:04,812 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39089-0x101456fc9c70001, quorum=127.0.0.1:63224, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-16T14:57:04,812 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38897-0x101456fc9c70002, quorum=127.0.0.1:63224, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-16T14:57:04,812 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39089-0x101456fc9c70001, quorum=127.0.0.1:63224, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-16T14:57:04,812 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38897-0x101456fc9c70002, quorum=127.0.0.1:63224, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-16T14:57:04,815 INFO [master/fc3fda29f660:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=fc3fda29f660,43615,1731769023046, sessionid=0x101456fc9c70000, setting cluster-up flag (Was=false) 2024-11-16T14:57:04,845 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42985-0x101456fc9c70003, quorum=127.0.0.1:63224, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-16T14:57:04,845 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39089-0x101456fc9c70001, quorum=127.0.0.1:63224, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-16T14:57:04,845 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38897-0x101456fc9c70002, quorum=127.0.0.1:63224, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-16T14:57:04,845 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43615-0x101456fc9c70000, quorum=127.0.0.1:63224, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-16T14:57:04,870 DEBUG [master/fc3fda29f660:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort 2024-11-16T14:57:04,874 DEBUG [master/fc3fda29f660:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=fc3fda29f660,43615,1731769023046 2024-11-16T14:57:04,895 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42985-0x101456fc9c70003, quorum=127.0.0.1:63224, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-16T14:57:04,895 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43615-0x101456fc9c70000, quorum=127.0.0.1:63224, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-16T14:57:04,895 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38897-0x101456fc9c70002, quorum=127.0.0.1:63224, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-16T14:57:04,895 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39089-0x101456fc9c70001, quorum=127.0.0.1:63224, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-16T14:57:04,920 DEBUG [master/fc3fda29f660:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort 2024-11-16T14:57:04,924 DEBUG [master/fc3fda29f660:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=fc3fda29f660,43615,1731769023046 2024-11-16T14:57:04,933 INFO [master/fc3fda29f660:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at hdfs://localhost:42223/user/jenkins/test-data/a8176cb1-2ebb-b530-b3c7-53c9e53c186e/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again 2024-11-16T14:57:04,984 INFO [RS:0;fc3fda29f660:39089 {}] regionserver.HRegionServer(746): ClusterId : a2ba0a3e-7753-4646-b6b2-e5ea1356a3cb 2024-11-16T14:57:04,984 INFO [RS:1;fc3fda29f660:38897 {}] regionserver.HRegionServer(746): ClusterId : a2ba0a3e-7753-4646-b6b2-e5ea1356a3cb 2024-11-16T14:57:04,984 INFO [RS:2;fc3fda29f660:42985 {}] regionserver.HRegionServer(746): ClusterId : a2ba0a3e-7753-4646-b6b2-e5ea1356a3cb 2024-11-16T14:57:04,986 DEBUG [RS:0;fc3fda29f660:39089 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-16T14:57:04,986 DEBUG [RS:1;fc3fda29f660:38897 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-16T14:57:04,986 DEBUG [RS:2;fc3fda29f660:42985 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-16T14:57:05,000 DEBUG [master/fc3fda29f660:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-16T14:57:05,006 DEBUG [RS:1;fc3fda29f660:38897 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-16T14:57:05,006 DEBUG [RS:0;fc3fda29f660:39089 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-16T14:57:05,006 DEBUG [RS:1;fc3fda29f660:38897 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-16T14:57:05,007 DEBUG [RS:2;fc3fda29f660:42985 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-16T14:57:05,007 DEBUG [RS:0;fc3fda29f660:39089 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-16T14:57:05,007 DEBUG [RS:2;fc3fda29f660:42985 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-16T14:57:05,011 INFO [master/fc3fda29f660:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2 2024-11-16T14:57:05,017 INFO [master/fc3fda29f660:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2024-11-16T14:57:05,021 DEBUG [RS:1;fc3fda29f660:38897 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-16T14:57:05,021 DEBUG [RS:2;fc3fda29f660:42985 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-16T14:57:05,021 DEBUG [RS:0;fc3fda29f660:39089 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-16T14:57:05,021 DEBUG [RS:0;fc3fda29f660:39089 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1a5637b3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=fc3fda29f660/172.17.0.2:0 2024-11-16T14:57:05,021 DEBUG [RS:1;fc3fda29f660:38897 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@37c492dd, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=fc3fda29f660/172.17.0.2:0 2024-11-16T14:57:05,021 DEBUG [RS:2;fc3fda29f660:42985 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@708a2068, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=fc3fda29f660/172.17.0.2:0 2024-11-16T14:57:05,023 DEBUG [master/fc3fda29f660:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: fc3fda29f660,43615,1731769023046 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2024-11-16T14:57:05,031 DEBUG [master/fc3fda29f660:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/fc3fda29f660:0, corePoolSize=5, maxPoolSize=5 2024-11-16T14:57:05,031 DEBUG [master/fc3fda29f660:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/fc3fda29f660:0, corePoolSize=5, maxPoolSize=5 2024-11-16T14:57:05,031 DEBUG [master/fc3fda29f660:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/fc3fda29f660:0, corePoolSize=5, maxPoolSize=5 2024-11-16T14:57:05,031 DEBUG [master/fc3fda29f660:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/fc3fda29f660:0, corePoolSize=5, maxPoolSize=5 2024-11-16T14:57:05,031 DEBUG [master/fc3fda29f660:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/fc3fda29f660:0, corePoolSize=10, maxPoolSize=10 2024-11-16T14:57:05,032 DEBUG [master/fc3fda29f660:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/fc3fda29f660:0, corePoolSize=1, maxPoolSize=1 2024-11-16T14:57:05,032 DEBUG [master/fc3fda29f660:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/fc3fda29f660:0, corePoolSize=2, maxPoolSize=2 2024-11-16T14:57:05,032 DEBUG [master/fc3fda29f660:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/fc3fda29f660:0, corePoolSize=1, maxPoolSize=1 2024-11-16T14:57:05,035 INFO [master/fc3fda29f660:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731769055035 2024-11-16T14:57:05,037 INFO [master/fc3fda29f660:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2024-11-16T14:57:05,038 INFO [master/fc3fda29f660:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2024-11-16T14:57:05,040 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-16T14:57:05,041 DEBUG [RS:0;fc3fda29f660:39089 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;fc3fda29f660:39089 2024-11-16T14:57:05,041 DEBUG [RS:2;fc3fda29f660:42985 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:2;fc3fda29f660:42985 2024-11-16T14:57:05,041 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region 2024-11-16T14:57:05,041 DEBUG [RS:1;fc3fda29f660:38897 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:1;fc3fda29f660:38897 2024-11-16T14:57:05,044 INFO [master/fc3fda29f660:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2024-11-16T14:57:05,045 INFO [master/fc3fda29f660:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2024-11-16T14:57:05,045 INFO [master/fc3fda29f660:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2024-11-16T14:57:05,045 INFO [RS:0;fc3fda29f660:39089 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-16T14:57:05,045 INFO [master/fc3fda29f660:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2024-11-16T14:57:05,045 INFO [RS:0;fc3fda29f660:39089 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-16T14:57:05,045 DEBUG [RS:0;fc3fda29f660:39089 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-16T14:57:05,046 INFO [RS:1;fc3fda29f660:38897 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-16T14:57:05,046 INFO [RS:2;fc3fda29f660:42985 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-16T14:57:05,046 INFO [RS:1;fc3fda29f660:38897 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-16T14:57:05,046 INFO [RS:2;fc3fda29f660:42985 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-16T14:57:05,046 DEBUG [RS:1;fc3fda29f660:38897 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-16T14:57:05,046 DEBUG [RS:2;fc3fda29f660:42985 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-16T14:57:05,048 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:57:05,048 INFO [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-11-16T14:57:05,048 INFO [RS:1;fc3fda29f660:38897 {}] regionserver.HRegionServer(2659): reportForDuty to master=fc3fda29f660,43615,1731769023046 with port=38897, startcode=1731769023879 2024-11-16T14:57:05,048 INFO [RS:0;fc3fda29f660:39089 {}] regionserver.HRegionServer(2659): reportForDuty to master=fc3fda29f660,43615,1731769023046 with port=39089, startcode=1731769023769 2024-11-16T14:57:05,048 INFO [RS:2;fc3fda29f660:42985 {}] regionserver.HRegionServer(2659): reportForDuty to master=fc3fda29f660,43615,1731769023046 with port=42985, startcode=1731769023951 2024-11-16T14:57:05,050 INFO [master/fc3fda29f660:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-16T14:57:05,054 INFO [master/fc3fda29f660:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2024-11-16T14:57:05,055 INFO [master/fc3fda29f660:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2024-11-16T14:57:05,056 INFO [master/fc3fda29f660:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2024-11-16T14:57:05,059 INFO [master/fc3fda29f660:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2024-11-16T14:57:05,059 INFO [master/fc3fda29f660:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2024-11-16T14:57:05,062 DEBUG [master/fc3fda29f660:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/fc3fda29f660:0:becomeActiveMaster-HFileCleaner.large.0-1731769025060,5,FailOnTimeoutGroup] 2024-11-16T14:57:05,063 DEBUG [RS:1;fc3fda29f660:38897 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-16T14:57:05,063 DEBUG [RS:2;fc3fda29f660:42985 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-16T14:57:05,063 DEBUG [RS:0;fc3fda29f660:39089 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-16T14:57:05,066 DEBUG [master/fc3fda29f660:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/fc3fda29f660:0:becomeActiveMaster-HFileCleaner.small.0-1731769025063,5,FailOnTimeoutGroup] 2024-11-16T14:57:05,067 INFO [master/fc3fda29f660:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-16T14:57:05,067 INFO [master/fc3fda29f660:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2024-11-16T14:57:05,068 INFO [master/fc3fda29f660:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2024-11-16T14:57:05,068 INFO [master/fc3fda29f660:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2024-11-16T14:57:05,072 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40641 is added to blk_1073741831_1007 (size=1321) 2024-11-16T14:57:05,073 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36531 is added to blk_1073741831_1007 (size=1321) 2024-11-16T14:57:05,073 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41671 is added to blk_1073741831_1007 (size=1321) 2024-11-16T14:57:05,075 INFO [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to hdfs://localhost:42223/user/jenkins/test-data/a8176cb1-2ebb-b530-b3c7-53c9e53c186e/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321 2024-11-16T14:57:05,076 INFO [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:42223/user/jenkins/test-data/a8176cb1-2ebb-b530-b3c7-53c9e53c186e 2024-11-16T14:57:05,117 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36531 is added to blk_1073741832_1008 (size=32) 2024-11-16T14:57:05,117 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41671 is added to blk_1073741832_1008 (size=32) 2024-11-16T14:57:05,119 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40641 is added to blk_1073741832_1008 (size=32) 2024-11-16T14:57:05,120 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40799, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins.hfs.1 (auth:SIMPLE), service=RegionServerStatusService 2024-11-16T14:57:05,120 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51981, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=RegionServerStatusService 2024-11-16T14:57:05,120 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37255, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins.hfs.2 (auth:SIMPLE), service=RegionServerStatusService 2024-11-16T14:57:05,121 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-16T14:57:05,127 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-16T14:57:05,127 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=43615 {}] master.ServerManager(363): Checking decommissioned status of RegionServer fc3fda29f660,38897,1731769023879 2024-11-16T14:57:05,129 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-16T14:57:05,130 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:57:05,130 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=43615 {}] master.ServerManager(517): Registering regionserver=fc3fda29f660,38897,1731769023879 2024-11-16T14:57:05,131 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-16T14:57:05,131 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-16T14:57:05,133 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-16T14:57:05,134 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:57:05,135 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-16T14:57:05,135 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-16T14:57:05,138 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-16T14:57:05,139 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:57:05,140 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-16T14:57:05,140 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-16T14:57:05,143 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=43615 {}] master.ServerManager(363): Checking decommissioned status of RegionServer fc3fda29f660,42985,1731769023951 2024-11-16T14:57:05,143 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-16T14:57:05,143 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=43615 {}] master.ServerManager(517): Registering regionserver=fc3fda29f660,42985,1731769023951 2024-11-16T14:57:05,144 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:57:05,146 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-16T14:57:05,147 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-16T14:57:05,149 DEBUG [RS:1;fc3fda29f660:38897 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:42223/user/jenkins/test-data/a8176cb1-2ebb-b530-b3c7-53c9e53c186e 2024-11-16T14:57:05,149 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42223/user/jenkins/test-data/a8176cb1-2ebb-b530-b3c7-53c9e53c186e/data/hbase/meta/1588230740 2024-11-16T14:57:05,149 DEBUG [RS:1;fc3fda29f660:38897 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:42223 2024-11-16T14:57:05,149 DEBUG [RS:1;fc3fda29f660:38897 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-16T14:57:05,150 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42223/user/jenkins/test-data/a8176cb1-2ebb-b530-b3c7-53c9e53c186e/data/hbase/meta/1588230740 2024-11-16T14:57:05,150 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=43615 {}] master.ServerManager(363): Checking decommissioned status of RegionServer fc3fda29f660,39089,1731769023769 2024-11-16T14:57:05,150 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=43615 {}] master.ServerManager(517): Registering regionserver=fc3fda29f660,39089,1731769023769 2024-11-16T14:57:05,151 DEBUG [RS:2;fc3fda29f660:42985 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:42223/user/jenkins/test-data/a8176cb1-2ebb-b530-b3c7-53c9e53c186e 2024-11-16T14:57:05,151 DEBUG [RS:2;fc3fda29f660:42985 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:42223 2024-11-16T14:57:05,151 DEBUG [RS:2;fc3fda29f660:42985 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-16T14:57:05,154 DEBUG [RS:0;fc3fda29f660:39089 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:42223/user/jenkins/test-data/a8176cb1-2ebb-b530-b3c7-53c9e53c186e 2024-11-16T14:57:05,155 DEBUG [RS:0;fc3fda29f660:39089 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:42223 2024-11-16T14:57:05,155 DEBUG [RS:0;fc3fda29f660:39089 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-16T14:57:05,155 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-16T14:57:05,155 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-16T14:57:05,156 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-16T14:57:05,159 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-16T14:57:05,164 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:42223/user/jenkins/test-data/a8176cb1-2ebb-b530-b3c7-53c9e53c186e/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-16T14:57:05,165 INFO [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=62218180, jitterRate=-0.07287687063217163}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-16T14:57:05,170 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740: Writing region info on filesystem at 1731769025121Initializing all the Stores at 1731769025122 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731769025123 (+1 ms)Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731769025126 (+3 ms)Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731769025126Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731769025127 (+1 ms)Cleaning up temporary data from old regions at 1731769025155 (+28 ms)Region opened successfully at 1731769025170 (+15 ms) 2024-11-16T14:57:05,171 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-16T14:57:05,171 INFO [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-16T14:57:05,171 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-16T14:57:05,171 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-16T14:57:05,171 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-16T14:57:05,173 INFO [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-16T14:57:05,173 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1731769025171Disabling compacts and flushes for region at 1731769025171Disabling writes for close at 1731769025171Writing region close event to WAL at 1731769025172 (+1 ms)Closed at 1731769025173 (+1 ms) 2024-11-16T14:57:05,176 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-16T14:57:05,176 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta 2024-11-16T14:57:05,178 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43615-0x101456fc9c70000, quorum=127.0.0.1:63224, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-16T14:57:05,184 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-11-16T14:57:05,196 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-16T14:57:05,199 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2024-11-16T14:57:05,210 DEBUG [RS:2;fc3fda29f660:42985 {}] zookeeper.ZKUtil(111): regionserver:42985-0x101456fc9c70003, quorum=127.0.0.1:63224, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/fc3fda29f660,42985,1731769023951 2024-11-16T14:57:05,211 WARN [RS:2;fc3fda29f660:42985 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-16T14:57:05,211 DEBUG [RS:0;fc3fda29f660:39089 {}] zookeeper.ZKUtil(111): regionserver:39089-0x101456fc9c70001, quorum=127.0.0.1:63224, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/fc3fda29f660,39089,1731769023769 2024-11-16T14:57:05,211 DEBUG [RS:1;fc3fda29f660:38897 {}] zookeeper.ZKUtil(111): regionserver:38897-0x101456fc9c70002, quorum=127.0.0.1:63224, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/fc3fda29f660,38897,1731769023879 2024-11-16T14:57:05,211 INFO [RS:2;fc3fda29f660:42985 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-16T14:57:05,211 WARN [RS:0;fc3fda29f660:39089 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-16T14:57:05,211 WARN [RS:1;fc3fda29f660:38897 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-16T14:57:05,211 INFO [RS:0;fc3fda29f660:39089 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-16T14:57:05,211 INFO [RS:1;fc3fda29f660:38897 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-16T14:57:05,211 DEBUG [RS:2;fc3fda29f660:42985 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:42223/user/jenkins/test-data/a8176cb1-2ebb-b530-b3c7-53c9e53c186e/WALs/fc3fda29f660,42985,1731769023951 2024-11-16T14:57:05,211 DEBUG [RS:0;fc3fda29f660:39089 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:42223/user/jenkins/test-data/a8176cb1-2ebb-b530-b3c7-53c9e53c186e/WALs/fc3fda29f660,39089,1731769023769 2024-11-16T14:57:05,211 DEBUG [RS:1;fc3fda29f660:38897 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:42223/user/jenkins/test-data/a8176cb1-2ebb-b530-b3c7-53c9e53c186e/WALs/fc3fda29f660,38897,1731769023879 2024-11-16T14:57:05,212 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [fc3fda29f660,42985,1731769023951] 2024-11-16T14:57:05,212 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [fc3fda29f660,39089,1731769023769] 2024-11-16T14:57:05,212 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [fc3fda29f660,38897,1731769023879] 2024-11-16T14:57:05,237 INFO [RS:1;fc3fda29f660:38897 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-16T14:57:05,237 INFO [RS:0;fc3fda29f660:39089 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-16T14:57:05,237 INFO [RS:2;fc3fda29f660:42985 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-16T14:57:05,254 INFO [RS:0;fc3fda29f660:39089 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-16T14:57:05,254 INFO [RS:2;fc3fda29f660:42985 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-16T14:57:05,254 INFO [RS:1;fc3fda29f660:38897 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-16T14:57:05,259 INFO [RS:2;fc3fda29f660:42985 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-16T14:57:05,259 INFO [RS:1;fc3fda29f660:38897 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-16T14:57:05,259 INFO [RS:0;fc3fda29f660:39089 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-16T14:57:05,259 INFO [RS:2;fc3fda29f660:42985 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-16T14:57:05,259 INFO [RS:0;fc3fda29f660:39089 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-16T14:57:05,259 INFO [RS:1;fc3fda29f660:38897 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-16T14:57:05,260 INFO [RS:2;fc3fda29f660:42985 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-16T14:57:05,262 INFO [RS:0;fc3fda29f660:39089 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-16T14:57:05,262 INFO [RS:1;fc3fda29f660:38897 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-16T14:57:05,265 INFO [RS:2;fc3fda29f660:42985 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-16T14:57:05,265 INFO [RS:0;fc3fda29f660:39089 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-16T14:57:05,265 INFO [RS:1;fc3fda29f660:38897 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-16T14:57:05,266 INFO [RS:2;fc3fda29f660:42985 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-16T14:57:05,266 INFO [RS:0;fc3fda29f660:39089 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-16T14:57:05,267 INFO [RS:1;fc3fda29f660:38897 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-16T14:57:05,267 DEBUG [RS:1;fc3fda29f660:38897 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/fc3fda29f660:0, corePoolSize=1, maxPoolSize=1 2024-11-16T14:57:05,267 DEBUG [RS:2;fc3fda29f660:42985 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/fc3fda29f660:0, corePoolSize=1, maxPoolSize=1 2024-11-16T14:57:05,267 DEBUG [RS:0;fc3fda29f660:39089 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/fc3fda29f660:0, corePoolSize=1, maxPoolSize=1 2024-11-16T14:57:05,267 DEBUG [RS:1;fc3fda29f660:38897 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/fc3fda29f660:0, corePoolSize=1, maxPoolSize=1 2024-11-16T14:57:05,267 DEBUG [RS:0;fc3fda29f660:39089 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/fc3fda29f660:0, corePoolSize=1, maxPoolSize=1 2024-11-16T14:57:05,267 DEBUG [RS:2;fc3fda29f660:42985 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/fc3fda29f660:0, corePoolSize=1, maxPoolSize=1 2024-11-16T14:57:05,267 DEBUG [RS:1;fc3fda29f660:38897 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/fc3fda29f660:0, corePoolSize=1, maxPoolSize=1 2024-11-16T14:57:05,267 DEBUG [RS:0;fc3fda29f660:39089 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/fc3fda29f660:0, corePoolSize=1, maxPoolSize=1 2024-11-16T14:57:05,267 DEBUG [RS:1;fc3fda29f660:38897 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/fc3fda29f660:0, corePoolSize=1, maxPoolSize=1 2024-11-16T14:57:05,267 DEBUG [RS:2;fc3fda29f660:42985 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/fc3fda29f660:0, corePoolSize=1, maxPoolSize=1 2024-11-16T14:57:05,267 DEBUG [RS:0;fc3fda29f660:39089 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/fc3fda29f660:0, corePoolSize=1, maxPoolSize=1 2024-11-16T14:57:05,267 DEBUG [RS:1;fc3fda29f660:38897 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/fc3fda29f660:0, corePoolSize=1, maxPoolSize=1 2024-11-16T14:57:05,267 DEBUG [RS:2;fc3fda29f660:42985 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/fc3fda29f660:0, corePoolSize=1, maxPoolSize=1 2024-11-16T14:57:05,267 DEBUG [RS:1;fc3fda29f660:38897 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/fc3fda29f660:0, corePoolSize=2, maxPoolSize=2 2024-11-16T14:57:05,267 DEBUG [RS:0;fc3fda29f660:39089 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/fc3fda29f660:0, corePoolSize=1, maxPoolSize=1 2024-11-16T14:57:05,268 DEBUG [RS:1;fc3fda29f660:38897 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/fc3fda29f660:0, corePoolSize=1, maxPoolSize=1 2024-11-16T14:57:05,268 DEBUG [RS:2;fc3fda29f660:42985 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/fc3fda29f660:0, corePoolSize=1, maxPoolSize=1 2024-11-16T14:57:05,268 DEBUG [RS:0;fc3fda29f660:39089 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/fc3fda29f660:0, corePoolSize=2, maxPoolSize=2 2024-11-16T14:57:05,268 DEBUG [RS:1;fc3fda29f660:38897 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/fc3fda29f660:0, corePoolSize=1, maxPoolSize=1 2024-11-16T14:57:05,268 DEBUG [RS:2;fc3fda29f660:42985 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/fc3fda29f660:0, corePoolSize=2, maxPoolSize=2 2024-11-16T14:57:05,268 DEBUG [RS:0;fc3fda29f660:39089 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/fc3fda29f660:0, corePoolSize=1, maxPoolSize=1 2024-11-16T14:57:05,268 DEBUG [RS:1;fc3fda29f660:38897 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/fc3fda29f660:0, corePoolSize=1, maxPoolSize=1 2024-11-16T14:57:05,268 DEBUG [RS:2;fc3fda29f660:42985 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/fc3fda29f660:0, corePoolSize=1, maxPoolSize=1 2024-11-16T14:57:05,268 DEBUG [RS:0;fc3fda29f660:39089 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/fc3fda29f660:0, corePoolSize=1, maxPoolSize=1 2024-11-16T14:57:05,268 DEBUG [RS:1;fc3fda29f660:38897 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/fc3fda29f660:0, corePoolSize=1, maxPoolSize=1 2024-11-16T14:57:05,268 DEBUG [RS:2;fc3fda29f660:42985 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/fc3fda29f660:0, corePoolSize=1, maxPoolSize=1 2024-11-16T14:57:05,268 DEBUG [RS:0;fc3fda29f660:39089 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/fc3fda29f660:0, corePoolSize=1, maxPoolSize=1 2024-11-16T14:57:05,268 DEBUG [RS:1;fc3fda29f660:38897 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/fc3fda29f660:0, corePoolSize=1, maxPoolSize=1 2024-11-16T14:57:05,268 DEBUG [RS:1;fc3fda29f660:38897 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/fc3fda29f660:0, corePoolSize=1, maxPoolSize=1 2024-11-16T14:57:05,268 DEBUG [RS:0;fc3fda29f660:39089 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/fc3fda29f660:0, corePoolSize=1, maxPoolSize=1 2024-11-16T14:57:05,268 DEBUG [RS:2;fc3fda29f660:42985 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/fc3fda29f660:0, corePoolSize=1, maxPoolSize=1 2024-11-16T14:57:05,268 DEBUG [RS:1;fc3fda29f660:38897 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/fc3fda29f660:0, corePoolSize=3, maxPoolSize=3 2024-11-16T14:57:05,268 DEBUG [RS:2;fc3fda29f660:42985 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/fc3fda29f660:0, corePoolSize=1, maxPoolSize=1 2024-11-16T14:57:05,268 DEBUG [RS:0;fc3fda29f660:39089 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/fc3fda29f660:0, corePoolSize=1, maxPoolSize=1 2024-11-16T14:57:05,268 DEBUG [RS:1;fc3fda29f660:38897 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0, corePoolSize=3, maxPoolSize=3 2024-11-16T14:57:05,268 DEBUG [RS:2;fc3fda29f660:42985 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/fc3fda29f660:0, corePoolSize=1, maxPoolSize=1 2024-11-16T14:57:05,268 DEBUG [RS:0;fc3fda29f660:39089 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/fc3fda29f660:0, corePoolSize=1, maxPoolSize=1 2024-11-16T14:57:05,269 DEBUG [RS:2;fc3fda29f660:42985 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/fc3fda29f660:0, corePoolSize=1, maxPoolSize=1 2024-11-16T14:57:05,269 DEBUG [RS:0;fc3fda29f660:39089 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/fc3fda29f660:0, corePoolSize=3, maxPoolSize=3 2024-11-16T14:57:05,269 DEBUG [RS:2;fc3fda29f660:42985 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/fc3fda29f660:0, corePoolSize=3, maxPoolSize=3 2024-11-16T14:57:05,269 DEBUG [RS:0;fc3fda29f660:39089 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0, corePoolSize=3, maxPoolSize=3 2024-11-16T14:57:05,269 DEBUG [RS:2;fc3fda29f660:42985 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/fc3fda29f660:0, corePoolSize=3, maxPoolSize=3 2024-11-16T14:57:05,274 INFO [RS:1;fc3fda29f660:38897 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-16T14:57:05,274 INFO [RS:0;fc3fda29f660:39089 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-16T14:57:05,274 INFO [RS:1;fc3fda29f660:38897 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-16T14:57:05,274 INFO [RS:0;fc3fda29f660:39089 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-16T14:57:05,275 INFO [RS:1;fc3fda29f660:38897 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-16T14:57:05,275 INFO [RS:0;fc3fda29f660:39089 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-16T14:57:05,275 INFO [RS:1;fc3fda29f660:38897 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-16T14:57:05,275 INFO [RS:0;fc3fda29f660:39089 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-16T14:57:05,275 INFO [RS:2;fc3fda29f660:42985 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-16T14:57:05,275 INFO [RS:1;fc3fda29f660:38897 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-16T14:57:05,275 INFO [RS:0;fc3fda29f660:39089 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-16T14:57:05,275 INFO [RS:2;fc3fda29f660:42985 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-16T14:57:05,275 INFO [RS:1;fc3fda29f660:38897 {}] hbase.ChoreService(168): Chore ScheduledChore name=fc3fda29f660,38897,1731769023879-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-16T14:57:05,275 INFO [RS:0;fc3fda29f660:39089 {}] hbase.ChoreService(168): Chore ScheduledChore name=fc3fda29f660,39089,1731769023769-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-16T14:57:05,275 INFO [RS:2;fc3fda29f660:42985 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-16T14:57:05,275 INFO [RS:2;fc3fda29f660:42985 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-16T14:57:05,275 INFO [RS:2;fc3fda29f660:42985 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-16T14:57:05,275 INFO [RS:2;fc3fda29f660:42985 {}] hbase.ChoreService(168): Chore ScheduledChore name=fc3fda29f660,42985,1731769023951-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-16T14:57:05,293 INFO [RS:0;fc3fda29f660:39089 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-16T14:57:05,295 INFO [RS:1;fc3fda29f660:38897 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-16T14:57:05,295 INFO [RS:0;fc3fda29f660:39089 {}] hbase.ChoreService(168): Chore ScheduledChore name=fc3fda29f660,39089,1731769023769-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-16T14:57:05,295 INFO [RS:1;fc3fda29f660:38897 {}] hbase.ChoreService(168): Chore ScheduledChore name=fc3fda29f660,38897,1731769023879-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-16T14:57:05,296 INFO [RS:0;fc3fda29f660:39089 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-16T14:57:05,296 INFO [RS:1;fc3fda29f660:38897 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-16T14:57:05,296 INFO [RS:0;fc3fda29f660:39089 {}] regionserver.Replication(171): fc3fda29f660,39089,1731769023769 started 2024-11-16T14:57:05,296 INFO [RS:1;fc3fda29f660:38897 {}] regionserver.Replication(171): fc3fda29f660,38897,1731769023879 started 2024-11-16T14:57:05,298 INFO [RS:2;fc3fda29f660:42985 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-16T14:57:05,298 INFO [RS:2;fc3fda29f660:42985 {}] hbase.ChoreService(168): Chore ScheduledChore name=fc3fda29f660,42985,1731769023951-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-16T14:57:05,299 INFO [RS:2;fc3fda29f660:42985 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-16T14:57:05,299 INFO [RS:2;fc3fda29f660:42985 {}] regionserver.Replication(171): fc3fda29f660,42985,1731769023951 started 2024-11-16T14:57:05,313 INFO [RS:0;fc3fda29f660:39089 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-16T14:57:05,314 INFO [RS:0;fc3fda29f660:39089 {}] regionserver.HRegionServer(1482): Serving as fc3fda29f660,39089,1731769023769, RpcServer on fc3fda29f660/172.17.0.2:39089, sessionid=0x101456fc9c70001 2024-11-16T14:57:05,314 DEBUG [RS:0;fc3fda29f660:39089 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-16T14:57:05,315 DEBUG [RS:0;fc3fda29f660:39089 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager fc3fda29f660,39089,1731769023769 2024-11-16T14:57:05,315 DEBUG [RS:0;fc3fda29f660:39089 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'fc3fda29f660,39089,1731769023769' 2024-11-16T14:57:05,315 DEBUG [RS:0;fc3fda29f660:39089 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-16T14:57:05,316 INFO [RS:1;fc3fda29f660:38897 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-16T14:57:05,316 DEBUG [RS:0;fc3fda29f660:39089 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-16T14:57:05,316 INFO [RS:1;fc3fda29f660:38897 {}] regionserver.HRegionServer(1482): Serving as fc3fda29f660,38897,1731769023879, RpcServer on fc3fda29f660/172.17.0.2:38897, sessionid=0x101456fc9c70002 2024-11-16T14:57:05,316 DEBUG [RS:1;fc3fda29f660:38897 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-16T14:57:05,316 DEBUG [RS:1;fc3fda29f660:38897 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager fc3fda29f660,38897,1731769023879 2024-11-16T14:57:05,316 DEBUG [RS:1;fc3fda29f660:38897 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'fc3fda29f660,38897,1731769023879' 2024-11-16T14:57:05,317 DEBUG [RS:1;fc3fda29f660:38897 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-16T14:57:05,317 DEBUG [RS:0;fc3fda29f660:39089 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-16T14:57:05,317 DEBUG [RS:0;fc3fda29f660:39089 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-16T14:57:05,317 DEBUG [RS:1;fc3fda29f660:38897 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-16T14:57:05,317 DEBUG [RS:0;fc3fda29f660:39089 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager fc3fda29f660,39089,1731769023769 2024-11-16T14:57:05,317 DEBUG [RS:0;fc3fda29f660:39089 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'fc3fda29f660,39089,1731769023769' 2024-11-16T14:57:05,317 DEBUG [RS:0;fc3fda29f660:39089 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-16T14:57:05,318 DEBUG [RS:1;fc3fda29f660:38897 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-16T14:57:05,318 DEBUG [RS:1;fc3fda29f660:38897 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-16T14:57:05,318 DEBUG [RS:0;fc3fda29f660:39089 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-16T14:57:05,318 DEBUG [RS:1;fc3fda29f660:38897 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager fc3fda29f660,38897,1731769023879 2024-11-16T14:57:05,318 DEBUG [RS:1;fc3fda29f660:38897 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'fc3fda29f660,38897,1731769023879' 2024-11-16T14:57:05,318 DEBUG [RS:1;fc3fda29f660:38897 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-16T14:57:05,318 INFO [RS:2;fc3fda29f660:42985 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-16T14:57:05,318 INFO [RS:2;fc3fda29f660:42985 {}] regionserver.HRegionServer(1482): Serving as fc3fda29f660,42985,1731769023951, RpcServer on fc3fda29f660/172.17.0.2:42985, sessionid=0x101456fc9c70003 2024-11-16T14:57:05,318 DEBUG [RS:0;fc3fda29f660:39089 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-16T14:57:05,318 DEBUG [RS:2;fc3fda29f660:42985 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-16T14:57:05,318 INFO [RS:0;fc3fda29f660:39089 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-16T14:57:05,318 DEBUG [RS:2;fc3fda29f660:42985 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager fc3fda29f660,42985,1731769023951 2024-11-16T14:57:05,318 INFO [RS:0;fc3fda29f660:39089 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-16T14:57:05,318 DEBUG [RS:2;fc3fda29f660:42985 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'fc3fda29f660,42985,1731769023951' 2024-11-16T14:57:05,318 DEBUG [RS:1;fc3fda29f660:38897 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-16T14:57:05,318 DEBUG [RS:2;fc3fda29f660:42985 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-16T14:57:05,319 DEBUG [RS:1;fc3fda29f660:38897 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-16T14:57:05,319 INFO [RS:1;fc3fda29f660:38897 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-16T14:57:05,319 DEBUG [RS:2;fc3fda29f660:42985 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-16T14:57:05,319 INFO [RS:1;fc3fda29f660:38897 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-16T14:57:05,319 DEBUG [RS:2;fc3fda29f660:42985 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-16T14:57:05,319 DEBUG [RS:2;fc3fda29f660:42985 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-16T14:57:05,319 DEBUG [RS:2;fc3fda29f660:42985 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager fc3fda29f660,42985,1731769023951 2024-11-16T14:57:05,319 DEBUG [RS:2;fc3fda29f660:42985 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'fc3fda29f660,42985,1731769023951' 2024-11-16T14:57:05,320 DEBUG [RS:2;fc3fda29f660:42985 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-16T14:57:05,320 DEBUG [RS:2;fc3fda29f660:42985 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-16T14:57:05,320 DEBUG [RS:2;fc3fda29f660:42985 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-16T14:57:05,320 INFO [RS:2;fc3fda29f660:42985 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-16T14:57:05,320 INFO [RS:2;fc3fda29f660:42985 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-16T14:57:05,350 WARN [fc3fda29f660:43615 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-16T14:57:05,427 INFO [RS:0;fc3fda29f660:39089 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-16T14:57:05,427 INFO [RS:2;fc3fda29f660:42985 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-16T14:57:05,427 INFO [RS:1;fc3fda29f660:38897 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-16T14:57:05,430 INFO [RS:2;fc3fda29f660:42985 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=fc3fda29f660%2C42985%2C1731769023951, suffix=, logDir=hdfs://localhost:42223/user/jenkins/test-data/a8176cb1-2ebb-b530-b3c7-53c9e53c186e/WALs/fc3fda29f660,42985,1731769023951, archiveDir=hdfs://localhost:42223/user/jenkins/test-data/a8176cb1-2ebb-b530-b3c7-53c9e53c186e/oldWALs, maxLogs=32 2024-11-16T14:57:05,430 INFO [RS:1;fc3fda29f660:38897 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=fc3fda29f660%2C38897%2C1731769023879, suffix=, logDir=hdfs://localhost:42223/user/jenkins/test-data/a8176cb1-2ebb-b530-b3c7-53c9e53c186e/WALs/fc3fda29f660,38897,1731769023879, archiveDir=hdfs://localhost:42223/user/jenkins/test-data/a8176cb1-2ebb-b530-b3c7-53c9e53c186e/oldWALs, maxLogs=32 2024-11-16T14:57:05,430 INFO [RS:0;fc3fda29f660:39089 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=fc3fda29f660%2C39089%2C1731769023769, suffix=, logDir=hdfs://localhost:42223/user/jenkins/test-data/a8176cb1-2ebb-b530-b3c7-53c9e53c186e/WALs/fc3fda29f660,39089,1731769023769, archiveDir=hdfs://localhost:42223/user/jenkins/test-data/a8176cb1-2ebb-b530-b3c7-53c9e53c186e/oldWALs, maxLogs=32 2024-11-16T14:57:05,447 DEBUG [RS:1;fc3fda29f660:38897 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/a8176cb1-2ebb-b530-b3c7-53c9e53c186e/WALs/fc3fda29f660,38897,1731769023879/fc3fda29f660%2C38897%2C1731769023879.1731769025434, exclude list is [], retry=0 2024-11-16T14:57:05,447 DEBUG [RS:2;fc3fda29f660:42985 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/a8176cb1-2ebb-b530-b3c7-53c9e53c186e/WALs/fc3fda29f660,42985,1731769023951/fc3fda29f660%2C42985%2C1731769023951.1731769025434, exclude list is [], retry=0 2024-11-16T14:57:05,450 DEBUG [RS:0;fc3fda29f660:39089 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/a8176cb1-2ebb-b530-b3c7-53c9e53c186e/WALs/fc3fda29f660,39089,1731769023769/fc3fda29f660%2C39089%2C1731769023769.1731769025435, exclude list is [], retry=0 2024-11-16T14:57:05,452 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41671,DS-cfbd562b-a0b9-4a9e-8390-a5133b0d2c73,DISK] 2024-11-16T14:57:05,452 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41671,DS-cfbd562b-a0b9-4a9e-8390-a5133b0d2c73,DISK] 2024-11-16T14:57:05,452 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40641,DS-cecddcef-3bfd-4cf3-a998-7e060881adbc,DISK] 2024-11-16T14:57:05,453 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40641,DS-cecddcef-3bfd-4cf3-a998-7e060881adbc,DISK] 2024-11-16T14:57:05,453 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36531,DS-69dfd3e9-4ae1-4c6e-b575-41b55f386dcc,DISK] 2024-11-16T14:57:05,453 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36531,DS-69dfd3e9-4ae1-4c6e-b575-41b55f386dcc,DISK] 2024-11-16T14:57:05,455 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40641,DS-cecddcef-3bfd-4cf3-a998-7e060881adbc,DISK] 2024-11-16T14:57:05,455 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36531,DS-69dfd3e9-4ae1-4c6e-b575-41b55f386dcc,DISK] 2024-11-16T14:57:05,455 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41671,DS-cfbd562b-a0b9-4a9e-8390-a5133b0d2c73,DISK] 2024-11-16T14:57:05,461 INFO [RS:1;fc3fda29f660:38897 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/a8176cb1-2ebb-b530-b3c7-53c9e53c186e/WALs/fc3fda29f660,38897,1731769023879/fc3fda29f660%2C38897%2C1731769023879.1731769025434 2024-11-16T14:57:05,461 INFO [RS:0;fc3fda29f660:39089 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/a8176cb1-2ebb-b530-b3c7-53c9e53c186e/WALs/fc3fda29f660,39089,1731769023769/fc3fda29f660%2C39089%2C1731769023769.1731769025435 2024-11-16T14:57:05,462 DEBUG [RS:1;fc3fda29f660:38897 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:40829:40829),(127.0.0.1/127.0.0.1:39249:39249),(127.0.0.1/127.0.0.1:35235:35235)] 2024-11-16T14:57:05,462 DEBUG [RS:0;fc3fda29f660:39089 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:40829:40829),(127.0.0.1/127.0.0.1:39249:39249),(127.0.0.1/127.0.0.1:35235:35235)] 2024-11-16T14:57:05,462 INFO [RS:2;fc3fda29f660:42985 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/a8176cb1-2ebb-b530-b3c7-53c9e53c186e/WALs/fc3fda29f660,42985,1731769023951/fc3fda29f660%2C42985%2C1731769023951.1731769025434 2024-11-16T14:57:05,463 DEBUG [RS:2;fc3fda29f660:42985 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:40829:40829),(127.0.0.1/127.0.0.1:35235:35235),(127.0.0.1/127.0.0.1:39249:39249)] 2024-11-16T14:57:05,604 DEBUG [fc3fda29f660:43615 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=3, allServersCount=3 2024-11-16T14:57:05,614 DEBUG [fc3fda29f660:43615 {}] balancer.BalancerClusterState(204): Hosts are {fc3fda29f660=0} racks are {/default-rack=0} 2024-11-16T14:57:05,620 DEBUG [fc3fda29f660:43615 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-11-16T14:57:05,620 DEBUG [fc3fda29f660:43615 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-11-16T14:57:05,620 DEBUG [fc3fda29f660:43615 {}] balancer.BalancerClusterState(303): server 2 has 0 regions 2024-11-16T14:57:05,620 DEBUG [fc3fda29f660:43615 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-11-16T14:57:05,620 DEBUG [fc3fda29f660:43615 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-11-16T14:57:05,620 DEBUG [fc3fda29f660:43615 {}] balancer.BalancerClusterState(310): server 2 is on host 0 2024-11-16T14:57:05,620 INFO [fc3fda29f660:43615 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-11-16T14:57:05,620 INFO [fc3fda29f660:43615 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-11-16T14:57:05,620 INFO [fc3fda29f660:43615 {}] balancer.BalancerClusterState(321): server 2 is on rack 0 2024-11-16T14:57:05,620 DEBUG [fc3fda29f660:43615 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-11-16T14:57:05,628 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=fc3fda29f660,38897,1731769023879 2024-11-16T14:57:05,635 INFO [PEWorker-1 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as fc3fda29f660,38897,1731769023879, state=OPENING 2024-11-16T14:57:05,670 DEBUG [PEWorker-1 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2024-11-16T14:57:05,678 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39089-0x101456fc9c70001, quorum=127.0.0.1:63224, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-16T14:57:05,678 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42985-0x101456fc9c70003, quorum=127.0.0.1:63224, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-16T14:57:05,678 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43615-0x101456fc9c70000, quorum=127.0.0.1:63224, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-16T14:57:05,678 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38897-0x101456fc9c70002, quorum=127.0.0.1:63224, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-16T14:57:05,679 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-16T14:57:05,679 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-16T14:57:05,679 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-16T14:57:05,679 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-16T14:57:05,682 DEBUG [PEWorker-1 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-16T14:57:05,686 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=fc3fda29f660,38897,1731769023879}] 2024-11-16T14:57:05,866 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-16T14:57:05,868 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-4-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46347, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-16T14:57:05,881 INFO [RS_OPEN_META-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-11-16T14:57:05,881 INFO [RS_OPEN_META-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-16T14:57:05,881 INFO [RS_OPEN_META-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-11-16T14:57:05,885 INFO [RS_OPEN_META-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=fc3fda29f660%2C38897%2C1731769023879.meta, suffix=.meta, logDir=hdfs://localhost:42223/user/jenkins/test-data/a8176cb1-2ebb-b530-b3c7-53c9e53c186e/WALs/fc3fda29f660,38897,1731769023879, archiveDir=hdfs://localhost:42223/user/jenkins/test-data/a8176cb1-2ebb-b530-b3c7-53c9e53c186e/oldWALs, maxLogs=32 2024-11-16T14:57:05,903 DEBUG [RS_OPEN_META-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_META, pid=3}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/a8176cb1-2ebb-b530-b3c7-53c9e53c186e/WALs/fc3fda29f660,38897,1731769023879/fc3fda29f660%2C38897%2C1731769023879.meta.1731769025887.meta, exclude list is [], retry=0 2024-11-16T14:57:05,907 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41671,DS-cfbd562b-a0b9-4a9e-8390-a5133b0d2c73,DISK] 2024-11-16T14:57:05,907 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36531,DS-69dfd3e9-4ae1-4c6e-b575-41b55f386dcc,DISK] 2024-11-16T14:57:05,907 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40641,DS-cecddcef-3bfd-4cf3-a998-7e060881adbc,DISK] 2024-11-16T14:57:05,910 INFO [RS_OPEN_META-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/a8176cb1-2ebb-b530-b3c7-53c9e53c186e/WALs/fc3fda29f660,38897,1731769023879/fc3fda29f660%2C38897%2C1731769023879.meta.1731769025887.meta 2024-11-16T14:57:05,910 DEBUG [RS_OPEN_META-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:40829:40829),(127.0.0.1/127.0.0.1:39249:39249),(127.0.0.1/127.0.0.1:35235:35235)] 2024-11-16T14:57:05,910 DEBUG [RS_OPEN_META-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-11-16T14:57:05,912 DEBUG [RS_OPEN_META-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-11-16T14:57:05,914 DEBUG [RS_OPEN_META-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-11-16T14:57:05,918 INFO [RS_OPEN_META-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-11-16T14:57:05,923 DEBUG [RS_OPEN_META-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-11-16T14:57:05,924 DEBUG [RS_OPEN_META-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-16T14:57:05,924 DEBUG [RS_OPEN_META-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-11-16T14:57:05,924 DEBUG [RS_OPEN_META-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-11-16T14:57:05,928 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-16T14:57:05,929 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-16T14:57:05,929 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:57:05,930 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-16T14:57:05,931 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-16T14:57:05,932 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-16T14:57:05,932 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:57:05,933 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-16T14:57:05,933 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-16T14:57:05,934 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-16T14:57:05,935 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:57:05,935 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-16T14:57:05,936 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-16T14:57:05,937 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-16T14:57:05,937 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:57:05,938 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-16T14:57:05,938 DEBUG [RS_OPEN_META-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-16T14:57:05,940 DEBUG [RS_OPEN_META-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42223/user/jenkins/test-data/a8176cb1-2ebb-b530-b3c7-53c9e53c186e/data/hbase/meta/1588230740 2024-11-16T14:57:05,943 DEBUG [RS_OPEN_META-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42223/user/jenkins/test-data/a8176cb1-2ebb-b530-b3c7-53c9e53c186e/data/hbase/meta/1588230740 2024-11-16T14:57:05,946 DEBUG [RS_OPEN_META-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-16T14:57:05,946 DEBUG [RS_OPEN_META-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-16T14:57:05,947 DEBUG [RS_OPEN_META-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-16T14:57:05,950 DEBUG [RS_OPEN_META-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-16T14:57:05,951 INFO [RS_OPEN_META-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=58983058, jitterRate=-0.12108394503593445}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-16T14:57:05,952 DEBUG [RS_OPEN_META-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-11-16T14:57:05,954 DEBUG [RS_OPEN_META-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1731769025925Writing region info on filesystem at 1731769025925Initializing all the Stores at 1731769025927 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731769025927Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731769025927Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731769025927Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731769025927Cleaning up temporary data from old regions at 1731769025946 (+19 ms)Running coprocessor post-open hooks at 1731769025952 (+6 ms)Region opened successfully at 1731769025953 (+1 ms) 2024-11-16T14:57:05,962 INFO [RS_OPEN_META-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1731769025855 2024-11-16T14:57:05,972 DEBUG [RS_OPEN_META-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-11-16T14:57:05,973 INFO [RS_OPEN_META-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-11-16T14:57:05,974 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=fc3fda29f660,38897,1731769023879 2024-11-16T14:57:05,976 INFO [PEWorker-1 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as fc3fda29f660,38897,1731769023879, state=OPEN 2024-11-16T14:57:06,078 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42985-0x101456fc9c70003, quorum=127.0.0.1:63224, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-16T14:57:06,078 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43615-0x101456fc9c70000, quorum=127.0.0.1:63224, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-16T14:57:06,078 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39089-0x101456fc9c70001, quorum=127.0.0.1:63224, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-16T14:57:06,078 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38897-0x101456fc9c70002, quorum=127.0.0.1:63224, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-16T14:57:06,079 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-16T14:57:06,079 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-16T14:57:06,079 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-16T14:57:06,079 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-16T14:57:06,080 DEBUG [PEWorker-1 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=fc3fda29f660,38897,1731769023879 2024-11-16T14:57:06,089 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2 2024-11-16T14:57:06,090 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=fc3fda29f660,38897,1731769023879 in 395 msec 2024-11-16T14:57:06,097 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1 2024-11-16T14:57:06,097 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 909 msec 2024-11-16T14:57:06,098 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-16T14:57:06,099 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces 2024-11-16T14:57:06,116 DEBUG [PEWorker-1 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-16T14:57:06,117 DEBUG [PEWorker-1 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fc3fda29f660,38897,1731769023879, seqNum=-1] 2024-11-16T14:57:06,134 DEBUG [PEWorker-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-16T14:57:06,136 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-4-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57781, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-16T14:57:06,156 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 1.1940 sec 2024-11-16T14:57:06,156 INFO [master/fc3fda29f660:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1731769026156, completionTime=-1 2024-11-16T14:57:06,158 INFO [master/fc3fda29f660:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=3; waited=0ms, expected min=3 server(s), max=3 server(s), master is running 2024-11-16T14:57:06,159 DEBUG [master/fc3fda29f660:0:becomeActiveMaster {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-16T14:57:06,183 INFO [master/fc3fda29f660:0:becomeActiveMaster {}] assignment.AssignmentManager(1776): Number of RegionServers=3 2024-11-16T14:57:06,183 INFO [master/fc3fda29f660:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731769086183 2024-11-16T14:57:06,183 INFO [master/fc3fda29f660:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731769146183 2024-11-16T14:57:06,183 INFO [master/fc3fda29f660:0:becomeActiveMaster {}] assignment.AssignmentManager(1783): Joined the cluster in 24 msec 2024-11-16T14:57:06,185 DEBUG [master/fc3fda29f660:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 1588230740 changed from -1.0 to 0.0, refreshing cache 2024-11-16T14:57:06,191 INFO [master/fc3fda29f660:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=fc3fda29f660,43615,1731769023046-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-16T14:57:06,191 INFO [master/fc3fda29f660:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=fc3fda29f660,43615,1731769023046-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-16T14:57:06,191 INFO [master/fc3fda29f660:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=fc3fda29f660,43615,1731769023046-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-16T14:57:06,193 INFO [master/fc3fda29f660:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-fc3fda29f660:43615, period=300000, unit=MILLISECONDS is enabled. 2024-11-16T14:57:06,193 INFO [master/fc3fda29f660:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2024-11-16T14:57:06,194 INFO [master/fc3fda29f660:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled. 2024-11-16T14:57:06,199 DEBUG [master/fc3fda29f660:0.Chore.1 {}] janitor.CatalogJanitor(180): 2024-11-16T14:57:06,219 INFO [master/fc3fda29f660:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 2.164sec 2024-11-16T14:57:06,221 INFO [master/fc3fda29f660:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2024-11-16T14:57:06,222 INFO [master/fc3fda29f660:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2024-11-16T14:57:06,223 INFO [master/fc3fda29f660:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2024-11-16T14:57:06,223 INFO [master/fc3fda29f660:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2024-11-16T14:57:06,223 INFO [master/fc3fda29f660:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2024-11-16T14:57:06,224 INFO [master/fc3fda29f660:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=fc3fda29f660,43615,1731769023046-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-16T14:57:06,224 INFO [master/fc3fda29f660:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=fc3fda29f660,43615,1731769023046-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2024-11-16T14:57:06,228 DEBUG [master/fc3fda29f660:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds 2024-11-16T14:57:06,229 INFO [master/fc3fda29f660:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2024-11-16T14:57:06,230 INFO [master/fc3fda29f660:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=fc3fda29f660,43615,1731769023046-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-16T14:57:06,294 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4b5e5329, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T14:57:06,298 DEBUG [Time-limited test {}] nio.NioEventLoop(110): -Dio.netty.noKeySetOptimization: false 2024-11-16T14:57:06,298 DEBUG [Time-limited test {}] nio.NioEventLoop(111): -Dio.netty.selectorAutoRebuildThreshold: 512 2024-11-16T14:57:06,302 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request fc3fda29f660,43615,-1 for getting cluster id 2024-11-16T14:57:06,304 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-16T14:57:06,313 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'a2ba0a3e-7753-4646-b6b2-e5ea1356a3cb' 2024-11-16T14:57:06,315 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-16T14:57:06,315 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "a2ba0a3e-7753-4646-b6b2-e5ea1356a3cb" 2024-11-16T14:57:06,317 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@65a292bc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T14:57:06,317 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [fc3fda29f660,43615,-1] 2024-11-16T14:57:06,319 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-16T14:57:06,321 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T14:57:06,322 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54456, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-16T14:57:06,325 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6c26e9a6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T14:57:06,325 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-16T14:57:06,332 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fc3fda29f660,38897,1731769023879, seqNum=-1] 2024-11-16T14:57:06,333 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-16T14:57:06,335 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-4-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52522, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-16T14:57:06,373 INFO [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=fc3fda29f660,43615,1731769023046 2024-11-16T14:57:06,375 WARN [Time-limited test {}] procedure2.ProcedureTestingUtility(193): Set Kill before store update to: false 2024-11-16T14:57:06,379 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-16T14:57:06,384 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] client.AsyncConnectionImpl(321): The fetched master address is fc3fda29f660,43615,1731769023046 2024-11-16T14:57:06,386 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@358db31d 2024-11-16T14:57:06,387 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-16T14:57:06,390 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54468, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-16T14:57:06,402 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43615 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'colfamily11', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'colfamily21', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'colfamily31', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-16T14:57:06,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43615 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true 2024-11-16T14:57:06,417 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true execute state=CREATE_TABLE_PRE_OPERATION 2024-11-16T14:57:06,420 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43615 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true" procId is: 4 2024-11-16T14:57:06,420 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:57:06,424 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-16T14:57:06,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43615 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-16T14:57:06,440 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41671 is added to blk_1073741837_1013 (size=1020) 2024-11-16T14:57:06,440 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36531 is added to blk_1073741837_1013 (size=1020) 2024-11-16T14:57:06,440 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40641 is added to blk_1073741837_1013 (size=1020) 2024-11-16T14:57:06,444 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 4f0d635e77a1e3dd04c4d107bb0f1564, NAME => 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,,1731769026392.4f0d635e77a1e3dd04c4d107bb0f1564.', STARTKEY => '', ENDKEY => 'bbb'}, tableDescriptor='testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'colfamily11', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'colfamily21', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'colfamily31', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:42223/user/jenkins/test-data/a8176cb1-2ebb-b530-b3c7-53c9e53c186e 2024-11-16T14:57:06,444 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-2 {}] regionserver.HRegion(7572): creating {ENCODED => 5bb22144956dcd43c9b1d957232b11e7, NAME => 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ccc,1731769026392.5bb22144956dcd43c9b1d957232b11e7.', STARTKEY => 'ccc', ENDKEY => 'ddd'}, tableDescriptor='testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'colfamily11', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'colfamily21', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'colfamily31', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:42223/user/jenkins/test-data/a8176cb1-2ebb-b530-b3c7-53c9e53c186e 2024-11-16T14:57:06,445 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-1 {}] regionserver.HRegion(7572): creating {ENCODED => c3dde5f5f668d914608a2987d4f4c001, NAME => 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,bbb,1731769026392.c3dde5f5f668d914608a2987d4f4c001.', STARTKEY => 'bbb', ENDKEY => 'ccc'}, tableDescriptor='testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'colfamily11', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'colfamily21', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'colfamily31', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:42223/user/jenkins/test-data/a8176cb1-2ebb-b530-b3c7-53c9e53c186e 2024-11-16T14:57:06,471 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36531 is added to blk_1073741839_1015 (size=110) 2024-11-16T14:57:06,471 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40641 is added to blk_1073741839_1015 (size=110) 2024-11-16T14:57:06,472 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41671 is added to blk_1073741839_1015 (size=110) 2024-11-16T14:57:06,472 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41671 is added to blk_1073741838_1014 (size=107) 2024-11-16T14:57:06,472 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36531 is added to blk_1073741838_1014 (size=107) 2024-11-16T14:57:06,473 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40641 is added to blk_1073741838_1014 (size=107) 2024-11-16T14:57:06,474 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36531 is added to blk_1073741840_1016 (size=110) 2024-11-16T14:57:06,474 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-1 {}] regionserver.HRegion(898): Instantiated testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,bbb,1731769026392.c3dde5f5f668d914608a2987d4f4c001.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-16T14:57:06,474 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-1 {}] regionserver.HRegion(1722): Closing c3dde5f5f668d914608a2987d4f4c001, disabling compactions & flushes 2024-11-16T14:57:06,474 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-1 {}] regionserver.HRegion(1755): Closing region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,bbb,1731769026392.c3dde5f5f668d914608a2987d4f4c001. 2024-11-16T14:57:06,474 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,bbb,1731769026392.c3dde5f5f668d914608a2987d4f4c001. 2024-11-16T14:57:06,474 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40641 is added to blk_1073741840_1016 (size=110) 2024-11-16T14:57:06,474 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-1 {}] regionserver.HRegion(1843): Acquired close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,bbb,1731769026392.c3dde5f5f668d914608a2987d4f4c001. after waiting 0 ms 2024-11-16T14:57:06,474 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-1 {}] regionserver.HRegion(1853): Updates disabled for region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,bbb,1731769026392.c3dde5f5f668d914608a2987d4f4c001. 2024-11-16T14:57:06,475 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-1 {}] regionserver.HRegion(1973): Closed testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,bbb,1731769026392.c3dde5f5f668d914608a2987d4f4c001. 2024-11-16T14:57:06,475 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-1 {}] regionserver.HRegion(1676): Region close journal for c3dde5f5f668d914608a2987d4f4c001: Waiting for close lock at 1731769026474Disabling compacts and flushes for region at 1731769026474Disabling writes for close at 1731769026474Writing region close event to WAL at 1731769026474Closed at 1731769026474 2024-11-16T14:57:06,475 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41671 is added to blk_1073741840_1016 (size=110) 2024-11-16T14:57:06,475 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(898): Instantiated testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,,1731769026392.4f0d635e77a1e3dd04c4d107bb0f1564.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-16T14:57:06,476 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-1 {}] regionserver.HRegion(7572): creating {ENCODED => dde7f54607f11c5fb4271e361e82dabd, NAME => 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ddd,1731769026392.dde7f54607f11c5fb4271e361e82dabd.', STARTKEY => 'ddd', ENDKEY => 'eee'}, tableDescriptor='testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'colfamily11', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'colfamily21', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'colfamily31', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:42223/user/jenkins/test-data/a8176cb1-2ebb-b530-b3c7-53c9e53c186e 2024-11-16T14:57:06,476 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(1722): Closing 4f0d635e77a1e3dd04c4d107bb0f1564, disabling compactions & flushes 2024-11-16T14:57:06,476 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(1755): Closing region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,,1731769026392.4f0d635e77a1e3dd04c4d107bb0f1564. 2024-11-16T14:57:06,476 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,,1731769026392.4f0d635e77a1e3dd04c4d107bb0f1564. 2024-11-16T14:57:06,476 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,,1731769026392.4f0d635e77a1e3dd04c4d107bb0f1564. after waiting 0 ms 2024-11-16T14:57:06,476 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,,1731769026392.4f0d635e77a1e3dd04c4d107bb0f1564. 2024-11-16T14:57:06,476 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(1973): Closed testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,,1731769026392.4f0d635e77a1e3dd04c4d107bb0f1564. 2024-11-16T14:57:06,476 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(1676): Region close journal for 4f0d635e77a1e3dd04c4d107bb0f1564: Waiting for close lock at 1731769026476Disabling compacts and flushes for region at 1731769026476Disabling writes for close at 1731769026476Writing region close event to WAL at 1731769026476Closed at 1731769026476 2024-11-16T14:57:06,477 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-2 {}] regionserver.HRegion(898): Instantiated testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ccc,1731769026392.5bb22144956dcd43c9b1d957232b11e7.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-16T14:57:06,477 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => fd3944fc2b78246a853fb2232c747c35, NAME => 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,eee,1731769026392.fd3944fc2b78246a853fb2232c747c35.', STARTKEY => 'eee', ENDKEY => 'fff'}, tableDescriptor='testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'colfamily11', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'colfamily21', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'colfamily31', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:42223/user/jenkins/test-data/a8176cb1-2ebb-b530-b3c7-53c9e53c186e 2024-11-16T14:57:06,477 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-2 {}] regionserver.HRegion(1722): Closing 5bb22144956dcd43c9b1d957232b11e7, disabling compactions & flushes 2024-11-16T14:57:06,477 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-2 {}] regionserver.HRegion(1755): Closing region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ccc,1731769026392.5bb22144956dcd43c9b1d957232b11e7. 2024-11-16T14:57:06,477 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-2 {}] regionserver.HRegion(1776): Time limited wait for close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ccc,1731769026392.5bb22144956dcd43c9b1d957232b11e7. 2024-11-16T14:57:06,477 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-2 {}] regionserver.HRegion(1843): Acquired close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ccc,1731769026392.5bb22144956dcd43c9b1d957232b11e7. after waiting 0 ms 2024-11-16T14:57:06,477 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-2 {}] regionserver.HRegion(1853): Updates disabled for region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ccc,1731769026392.5bb22144956dcd43c9b1d957232b11e7. 2024-11-16T14:57:06,477 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-2 {}] regionserver.HRegion(1973): Closed testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ccc,1731769026392.5bb22144956dcd43c9b1d957232b11e7. 2024-11-16T14:57:06,477 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-2 {}] regionserver.HRegion(1676): Region close journal for 5bb22144956dcd43c9b1d957232b11e7: Waiting for close lock at 1731769026477Disabling compacts and flushes for region at 1731769026477Disabling writes for close at 1731769026477Writing region close event to WAL at 1731769026477Closed at 1731769026477 2024-11-16T14:57:06,478 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-2 {}] regionserver.HRegion(7572): creating {ENCODED => a5db9431e22152bd2a82e84504d2879a, NAME => 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,fff,1731769026392.a5db9431e22152bd2a82e84504d2879a.', STARTKEY => 'fff', ENDKEY => 'ggg'}, tableDescriptor='testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'colfamily11', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'colfamily21', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'colfamily31', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:42223/user/jenkins/test-data/a8176cb1-2ebb-b530-b3c7-53c9e53c186e 2024-11-16T14:57:06,489 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36531 is added to blk_1073741843_1019 (size=110) 2024-11-16T14:57:06,489 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41671 is added to blk_1073741842_1018 (size=110) 2024-11-16T14:57:06,490 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40641 is added to blk_1073741843_1019 (size=110) 2024-11-16T14:57:06,490 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40641 is added to blk_1073741842_1018 (size=110) 2024-11-16T14:57:06,490 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36531 is added to blk_1073741842_1018 (size=110) 2024-11-16T14:57:06,490 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41671 is added to blk_1073741843_1019 (size=110) 2024-11-16T14:57:06,491 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-2 {}] regionserver.HRegion(898): Instantiated testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,fff,1731769026392.a5db9431e22152bd2a82e84504d2879a.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-16T14:57:06,492 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-2 {}] regionserver.HRegion(1722): Closing a5db9431e22152bd2a82e84504d2879a, disabling compactions & flushes 2024-11-16T14:57:06,492 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-2 {}] regionserver.HRegion(1755): Closing region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,fff,1731769026392.a5db9431e22152bd2a82e84504d2879a. 2024-11-16T14:57:06,492 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-2 {}] regionserver.HRegion(1776): Time limited wait for close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,fff,1731769026392.a5db9431e22152bd2a82e84504d2879a. 2024-11-16T14:57:06,492 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-2 {}] regionserver.HRegion(1843): Acquired close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,fff,1731769026392.a5db9431e22152bd2a82e84504d2879a. after waiting 0 ms 2024-11-16T14:57:06,492 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-2 {}] regionserver.HRegion(1853): Updates disabled for region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,fff,1731769026392.a5db9431e22152bd2a82e84504d2879a. 2024-11-16T14:57:06,492 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-2 {}] regionserver.HRegion(1973): Closed testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,fff,1731769026392.a5db9431e22152bd2a82e84504d2879a. 2024-11-16T14:57:06,492 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-2 {}] regionserver.HRegion(1676): Region close journal for a5db9431e22152bd2a82e84504d2879a: Waiting for close lock at 1731769026492Disabling compacts and flushes for region at 1731769026492Disabling writes for close at 1731769026492Writing region close event to WAL at 1731769026492Closed at 1731769026492 2024-11-16T14:57:06,493 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-2 {}] regionserver.HRegion(7572): creating {ENCODED => dcd6414ddebad66e87ab37a6dba924f3, NAME => 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ggg,1731769026392.dcd6414ddebad66e87ab37a6dba924f3.', STARTKEY => 'ggg', ENDKEY => 'hhh'}, tableDescriptor='testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'colfamily11', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'colfamily21', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'colfamily31', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:42223/user/jenkins/test-data/a8176cb1-2ebb-b530-b3c7-53c9e53c186e 2024-11-16T14:57:06,494 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(898): Instantiated testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,eee,1731769026392.fd3944fc2b78246a853fb2232c747c35.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-16T14:57:06,494 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(1722): Closing fd3944fc2b78246a853fb2232c747c35, disabling compactions & flushes 2024-11-16T14:57:06,494 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(1755): Closing region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,eee,1731769026392.fd3944fc2b78246a853fb2232c747c35. 2024-11-16T14:57:06,494 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,eee,1731769026392.fd3944fc2b78246a853fb2232c747c35. 2024-11-16T14:57:06,494 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,eee,1731769026392.fd3944fc2b78246a853fb2232c747c35. after waiting 0 ms 2024-11-16T14:57:06,494 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,eee,1731769026392.fd3944fc2b78246a853fb2232c747c35. 2024-11-16T14:57:06,494 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(1973): Closed testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,eee,1731769026392.fd3944fc2b78246a853fb2232c747c35. 2024-11-16T14:57:06,494 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(1676): Region close journal for fd3944fc2b78246a853fb2232c747c35: Waiting for close lock at 1731769026494Disabling compacts and flushes for region at 1731769026494Disabling writes for close at 1731769026494Writing region close event to WAL at 1731769026494Closed at 1731769026494 2024-11-16T14:57:06,495 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 1d97df67e5110b0fe1a1e4b7c1341cbb, NAME => 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,hhh,1731769026392.1d97df67e5110b0fe1a1e4b7c1341cbb.', STARTKEY => 'hhh', ENDKEY => 'iii'}, tableDescriptor='testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'colfamily11', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'colfamily21', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'colfamily31', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:42223/user/jenkins/test-data/a8176cb1-2ebb-b530-b3c7-53c9e53c186e 2024-11-16T14:57:06,500 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36531 is added to blk_1073741841_1017 (size=110) 2024-11-16T14:57:06,500 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40641 is added to blk_1073741841_1017 (size=110) 2024-11-16T14:57:06,501 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41671 is added to blk_1073741841_1017 (size=110) 2024-11-16T14:57:06,502 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-1 {}] regionserver.HRegion(898): Instantiated testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ddd,1731769026392.dde7f54607f11c5fb4271e361e82dabd.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-16T14:57:06,502 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-1 {}] regionserver.HRegion(1722): Closing dde7f54607f11c5fb4271e361e82dabd, disabling compactions & flushes 2024-11-16T14:57:06,502 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-1 {}] regionserver.HRegion(1755): Closing region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ddd,1731769026392.dde7f54607f11c5fb4271e361e82dabd. 2024-11-16T14:57:06,502 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ddd,1731769026392.dde7f54607f11c5fb4271e361e82dabd. 2024-11-16T14:57:06,503 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-1 {}] regionserver.HRegion(1843): Acquired close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ddd,1731769026392.dde7f54607f11c5fb4271e361e82dabd. after waiting 0 ms 2024-11-16T14:57:06,503 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-1 {}] regionserver.HRegion(1853): Updates disabled for region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ddd,1731769026392.dde7f54607f11c5fb4271e361e82dabd. 2024-11-16T14:57:06,503 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-1 {}] regionserver.HRegion(1973): Closed testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ddd,1731769026392.dde7f54607f11c5fb4271e361e82dabd. 2024-11-16T14:57:06,503 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-1 {}] regionserver.HRegion(1676): Region close journal for dde7f54607f11c5fb4271e361e82dabd: Waiting for close lock at 1731769026502Disabling compacts and flushes for region at 1731769026502Disabling writes for close at 1731769026503 (+1 ms)Writing region close event to WAL at 1731769026503Closed at 1731769026503 2024-11-16T14:57:06,503 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-1 {}] regionserver.HRegion(7572): creating {ENCODED => 7352dcdbf9811dcebc51a714a1de4f55, NAME => 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,iii,1731769026392.7352dcdbf9811dcebc51a714a1de4f55.', STARTKEY => 'iii', ENDKEY => 'jjj'}, tableDescriptor='testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'colfamily11', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'colfamily21', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'colfamily31', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:42223/user/jenkins/test-data/a8176cb1-2ebb-b530-b3c7-53c9e53c186e 2024-11-16T14:57:06,506 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40641 is added to blk_1073741844_1020 (size=110) 2024-11-16T14:57:06,506 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41671 is added to blk_1073741844_1020 (size=110) 2024-11-16T14:57:06,506 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36531 is added to blk_1073741844_1020 (size=110) 2024-11-16T14:57:06,517 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41671 is added to blk_1073741845_1021 (size=110) 2024-11-16T14:57:06,518 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40641 is added to blk_1073741845_1021 (size=110) 2024-11-16T14:57:06,519 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36531 is added to blk_1073741845_1021 (size=110) 2024-11-16T14:57:06,520 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(898): Instantiated testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,hhh,1731769026392.1d97df67e5110b0fe1a1e4b7c1341cbb.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-16T14:57:06,520 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(1722): Closing 1d97df67e5110b0fe1a1e4b7c1341cbb, disabling compactions & flushes 2024-11-16T14:57:06,520 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(1755): Closing region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,hhh,1731769026392.1d97df67e5110b0fe1a1e4b7c1341cbb. 2024-11-16T14:57:06,520 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,hhh,1731769026392.1d97df67e5110b0fe1a1e4b7c1341cbb. 2024-11-16T14:57:06,520 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,hhh,1731769026392.1d97df67e5110b0fe1a1e4b7c1341cbb. after waiting 0 ms 2024-11-16T14:57:06,520 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,hhh,1731769026392.1d97df67e5110b0fe1a1e4b7c1341cbb. 2024-11-16T14:57:06,520 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(1973): Closed testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,hhh,1731769026392.1d97df67e5110b0fe1a1e4b7c1341cbb. 2024-11-16T14:57:06,520 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(1676): Region close journal for 1d97df67e5110b0fe1a1e4b7c1341cbb: Waiting for close lock at 1731769026520Disabling compacts and flushes for region at 1731769026520Disabling writes for close at 1731769026520Writing region close event to WAL at 1731769026520Closed at 1731769026520 2024-11-16T14:57:06,521 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 7eb7b75d28b808c5c3f2f96e407b4347, NAME => 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,jjj,1731769026392.7eb7b75d28b808c5c3f2f96e407b4347.', STARTKEY => 'jjj', ENDKEY => 'kkk'}, tableDescriptor='testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'colfamily11', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'colfamily21', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'colfamily31', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:42223/user/jenkins/test-data/a8176cb1-2ebb-b530-b3c7-53c9e53c186e 2024-11-16T14:57:06,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43615 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-16T14:57:06,535 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36531 is added to blk_1073741846_1022 (size=110) 2024-11-16T14:57:06,535 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40641 is added to blk_1073741846_1022 (size=110) 2024-11-16T14:57:06,535 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41671 is added to blk_1073741846_1022 (size=110) 2024-11-16T14:57:06,540 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36531 is added to blk_1073741847_1023 (size=110) 2024-11-16T14:57:06,541 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40641 is added to blk_1073741847_1023 (size=110) 2024-11-16T14:57:06,541 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41671 is added to blk_1073741847_1023 (size=110) 2024-11-16T14:57:06,542 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(898): Instantiated testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,jjj,1731769026392.7eb7b75d28b808c5c3f2f96e407b4347.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-16T14:57:06,542 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(1722): Closing 7eb7b75d28b808c5c3f2f96e407b4347, disabling compactions & flushes 2024-11-16T14:57:06,542 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(1755): Closing region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,jjj,1731769026392.7eb7b75d28b808c5c3f2f96e407b4347. 2024-11-16T14:57:06,542 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,jjj,1731769026392.7eb7b75d28b808c5c3f2f96e407b4347. 2024-11-16T14:57:06,542 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,jjj,1731769026392.7eb7b75d28b808c5c3f2f96e407b4347. after waiting 0 ms 2024-11-16T14:57:06,542 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,jjj,1731769026392.7eb7b75d28b808c5c3f2f96e407b4347. 2024-11-16T14:57:06,542 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(1973): Closed testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,jjj,1731769026392.7eb7b75d28b808c5c3f2f96e407b4347. 2024-11-16T14:57:06,542 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(1676): Region close journal for 7eb7b75d28b808c5c3f2f96e407b4347: Waiting for close lock at 1731769026542Disabling compacts and flushes for region at 1731769026542Disabling writes for close at 1731769026542Writing region close event to WAL at 1731769026542Closed at 1731769026542 2024-11-16T14:57:06,543 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => b4f73326a2a6a279d503e557c6ed452f, NAME => 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,kkk,1731769026392.b4f73326a2a6a279d503e557c6ed452f.', STARTKEY => 'kkk', ENDKEY => 'lll'}, tableDescriptor='testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'colfamily11', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'colfamily21', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'colfamily31', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:42223/user/jenkins/test-data/a8176cb1-2ebb-b530-b3c7-53c9e53c186e 2024-11-16T14:57:06,557 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40641 is added to blk_1073741848_1024 (size=110) 2024-11-16T14:57:06,557 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36531 is added to blk_1073741848_1024 (size=110) 2024-11-16T14:57:06,558 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41671 is added to blk_1073741848_1024 (size=110) 2024-11-16T14:57:06,559 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(898): Instantiated testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,kkk,1731769026392.b4f73326a2a6a279d503e557c6ed452f.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-16T14:57:06,559 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(1722): Closing b4f73326a2a6a279d503e557c6ed452f, disabling compactions & flushes 2024-11-16T14:57:06,559 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(1755): Closing region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,kkk,1731769026392.b4f73326a2a6a279d503e557c6ed452f. 2024-11-16T14:57:06,559 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,kkk,1731769026392.b4f73326a2a6a279d503e557c6ed452f. 2024-11-16T14:57:06,559 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,kkk,1731769026392.b4f73326a2a6a279d503e557c6ed452f. after waiting 0 ms 2024-11-16T14:57:06,559 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,kkk,1731769026392.b4f73326a2a6a279d503e557c6ed452f. 2024-11-16T14:57:06,559 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(1973): Closed testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,kkk,1731769026392.b4f73326a2a6a279d503e557c6ed452f. 2024-11-16T14:57:06,559 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(1676): Region close journal for b4f73326a2a6a279d503e557c6ed452f: Waiting for close lock at 1731769026559Disabling compacts and flushes for region at 1731769026559Disabling writes for close at 1731769026559Writing region close event to WAL at 1731769026559Closed at 1731769026559 2024-11-16T14:57:06,560 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 266565e60d7419a2a30ceec1b0f6519b, NAME => 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,lll,1731769026392.266565e60d7419a2a30ceec1b0f6519b.', STARTKEY => 'lll', ENDKEY => 'mmm'}, tableDescriptor='testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'colfamily11', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'colfamily21', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'colfamily31', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:42223/user/jenkins/test-data/a8176cb1-2ebb-b530-b3c7-53c9e53c186e 2024-11-16T14:57:06,572 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40641 is added to blk_1073741849_1025 (size=110) 2024-11-16T14:57:06,572 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41671 is added to blk_1073741849_1025 (size=110) 2024-11-16T14:57:06,573 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36531 is added to blk_1073741849_1025 (size=110) 2024-11-16T14:57:06,574 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(898): Instantiated testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,lll,1731769026392.266565e60d7419a2a30ceec1b0f6519b.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-16T14:57:06,574 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(1722): Closing 266565e60d7419a2a30ceec1b0f6519b, disabling compactions & flushes 2024-11-16T14:57:06,574 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(1755): Closing region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,lll,1731769026392.266565e60d7419a2a30ceec1b0f6519b. 2024-11-16T14:57:06,574 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,lll,1731769026392.266565e60d7419a2a30ceec1b0f6519b. 2024-11-16T14:57:06,575 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,lll,1731769026392.266565e60d7419a2a30ceec1b0f6519b. after waiting 0 ms 2024-11-16T14:57:06,575 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,lll,1731769026392.266565e60d7419a2a30ceec1b0f6519b. 2024-11-16T14:57:06,575 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(1973): Closed testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,lll,1731769026392.266565e60d7419a2a30ceec1b0f6519b. 2024-11-16T14:57:06,575 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(1676): Region close journal for 266565e60d7419a2a30ceec1b0f6519b: Waiting for close lock at 1731769026574Disabling compacts and flushes for region at 1731769026574Disabling writes for close at 1731769026575 (+1 ms)Writing region close event to WAL at 1731769026575Closed at 1731769026575 2024-11-16T14:57:06,576 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => c2acbc05ae7bb1607a5719b701b40383, NAME => 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,mmm,1731769026392.c2acbc05ae7bb1607a5719b701b40383.', STARTKEY => 'mmm', ENDKEY => 'nnn'}, tableDescriptor='testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'colfamily11', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'colfamily21', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'colfamily31', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:42223/user/jenkins/test-data/a8176cb1-2ebb-b530-b3c7-53c9e53c186e 2024-11-16T14:57:06,587 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36531 is added to blk_1073741850_1026 (size=110) 2024-11-16T14:57:06,588 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41671 is added to blk_1073741850_1026 (size=110) 2024-11-16T14:57:06,588 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40641 is added to blk_1073741850_1026 (size=110) 2024-11-16T14:57:06,589 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(898): Instantiated testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,mmm,1731769026392.c2acbc05ae7bb1607a5719b701b40383.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-16T14:57:06,589 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(1722): Closing c2acbc05ae7bb1607a5719b701b40383, disabling compactions & flushes 2024-11-16T14:57:06,589 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(1755): Closing region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,mmm,1731769026392.c2acbc05ae7bb1607a5719b701b40383. 2024-11-16T14:57:06,589 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,mmm,1731769026392.c2acbc05ae7bb1607a5719b701b40383. 2024-11-16T14:57:06,589 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,mmm,1731769026392.c2acbc05ae7bb1607a5719b701b40383. after waiting 0 ms 2024-11-16T14:57:06,589 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,mmm,1731769026392.c2acbc05ae7bb1607a5719b701b40383. 2024-11-16T14:57:06,589 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(1973): Closed testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,mmm,1731769026392.c2acbc05ae7bb1607a5719b701b40383. 2024-11-16T14:57:06,589 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(1676): Region close journal for c2acbc05ae7bb1607a5719b701b40383: Waiting for close lock at 1731769026589Disabling compacts and flushes for region at 1731769026589Disabling writes for close at 1731769026589Writing region close event to WAL at 1731769026589Closed at 1731769026589 2024-11-16T14:57:06,590 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 994ca1526cf6c4838e65f85444853f1a, NAME => 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,nnn,1731769026392.994ca1526cf6c4838e65f85444853f1a.', STARTKEY => 'nnn', ENDKEY => 'ooo'}, tableDescriptor='testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'colfamily11', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'colfamily21', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'colfamily31', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:42223/user/jenkins/test-data/a8176cb1-2ebb-b530-b3c7-53c9e53c186e 2024-11-16T14:57:06,601 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36531 is added to blk_1073741851_1027 (size=110) 2024-11-16T14:57:06,601 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40641 is added to blk_1073741851_1027 (size=110) 2024-11-16T14:57:06,601 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41671 is added to blk_1073741851_1027 (size=110) 2024-11-16T14:57:06,602 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(898): Instantiated testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,nnn,1731769026392.994ca1526cf6c4838e65f85444853f1a.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-16T14:57:06,602 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(1722): Closing 994ca1526cf6c4838e65f85444853f1a, disabling compactions & flushes 2024-11-16T14:57:06,602 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(1755): Closing region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,nnn,1731769026392.994ca1526cf6c4838e65f85444853f1a. 2024-11-16T14:57:06,602 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,nnn,1731769026392.994ca1526cf6c4838e65f85444853f1a. 2024-11-16T14:57:06,602 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,nnn,1731769026392.994ca1526cf6c4838e65f85444853f1a. after waiting 0 ms 2024-11-16T14:57:06,603 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,nnn,1731769026392.994ca1526cf6c4838e65f85444853f1a. 2024-11-16T14:57:06,603 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(1973): Closed testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,nnn,1731769026392.994ca1526cf6c4838e65f85444853f1a. 2024-11-16T14:57:06,603 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(1676): Region close journal for 994ca1526cf6c4838e65f85444853f1a: Waiting for close lock at 1731769026602Disabling compacts and flushes for region at 1731769026602Disabling writes for close at 1731769026602Writing region close event to WAL at 1731769026603 (+1 ms)Closed at 1731769026603 2024-11-16T14:57:06,604 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 299131947f15f9c44bcfe22cd6623096, NAME => 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ooo,1731769026392.299131947f15f9c44bcfe22cd6623096.', STARTKEY => 'ooo', ENDKEY => 'ppp'}, tableDescriptor='testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'colfamily11', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'colfamily21', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'colfamily31', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:42223/user/jenkins/test-data/a8176cb1-2ebb-b530-b3c7-53c9e53c186e 2024-11-16T14:57:06,612 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41671 is added to blk_1073741852_1028 (size=110) 2024-11-16T14:57:06,613 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36531 is added to blk_1073741852_1028 (size=110) 2024-11-16T14:57:06,613 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40641 is added to blk_1073741852_1028 (size=110) 2024-11-16T14:57:06,614 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(898): Instantiated testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ooo,1731769026392.299131947f15f9c44bcfe22cd6623096.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-16T14:57:06,614 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(1722): Closing 299131947f15f9c44bcfe22cd6623096, disabling compactions & flushes 2024-11-16T14:57:06,614 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(1755): Closing region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ooo,1731769026392.299131947f15f9c44bcfe22cd6623096. 2024-11-16T14:57:06,614 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ooo,1731769026392.299131947f15f9c44bcfe22cd6623096. 2024-11-16T14:57:06,614 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ooo,1731769026392.299131947f15f9c44bcfe22cd6623096. after waiting 0 ms 2024-11-16T14:57:06,614 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ooo,1731769026392.299131947f15f9c44bcfe22cd6623096. 2024-11-16T14:57:06,614 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(1973): Closed testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ooo,1731769026392.299131947f15f9c44bcfe22cd6623096. 2024-11-16T14:57:06,614 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(1676): Region close journal for 299131947f15f9c44bcfe22cd6623096: Waiting for close lock at 1731769026614Disabling compacts and flushes for region at 1731769026614Disabling writes for close at 1731769026614Writing region close event to WAL at 1731769026614Closed at 1731769026614 2024-11-16T14:57:06,615 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 7f5c9df3a544933227f58b052f743f72, NAME => 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ppp,1731769026392.7f5c9df3a544933227f58b052f743f72.', STARTKEY => 'ppp', ENDKEY => 'qqq'}, tableDescriptor='testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'colfamily11', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'colfamily21', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'colfamily31', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:42223/user/jenkins/test-data/a8176cb1-2ebb-b530-b3c7-53c9e53c186e 2024-11-16T14:57:06,626 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41671 is added to blk_1073741853_1029 (size=110) 2024-11-16T14:57:06,626 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40641 is added to blk_1073741853_1029 (size=110) 2024-11-16T14:57:06,627 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36531 is added to blk_1073741853_1029 (size=110) 2024-11-16T14:57:06,628 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(898): Instantiated testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ppp,1731769026392.7f5c9df3a544933227f58b052f743f72.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-16T14:57:06,628 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(1722): Closing 7f5c9df3a544933227f58b052f743f72, disabling compactions & flushes 2024-11-16T14:57:06,628 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(1755): Closing region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ppp,1731769026392.7f5c9df3a544933227f58b052f743f72. 2024-11-16T14:57:06,628 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ppp,1731769026392.7f5c9df3a544933227f58b052f743f72. 2024-11-16T14:57:06,628 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ppp,1731769026392.7f5c9df3a544933227f58b052f743f72. after waiting 0 ms 2024-11-16T14:57:06,628 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ppp,1731769026392.7f5c9df3a544933227f58b052f743f72. 2024-11-16T14:57:06,628 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(1973): Closed testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ppp,1731769026392.7f5c9df3a544933227f58b052f743f72. 2024-11-16T14:57:06,628 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(1676): Region close journal for 7f5c9df3a544933227f58b052f743f72: Waiting for close lock at 1731769026628Disabling compacts and flushes for region at 1731769026628Disabling writes for close at 1731769026628Writing region close event to WAL at 1731769026628Closed at 1731769026628 2024-11-16T14:57:06,629 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 18f03e2509a3a89e4a69ba87fad73940, NAME => 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,qqq,1731769026392.18f03e2509a3a89e4a69ba87fad73940.', STARTKEY => 'qqq', ENDKEY => 'rrr'}, tableDescriptor='testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'colfamily11', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'colfamily21', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'colfamily31', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:42223/user/jenkins/test-data/a8176cb1-2ebb-b530-b3c7-53c9e53c186e 2024-11-16T14:57:06,639 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41671 is added to blk_1073741854_1030 (size=110) 2024-11-16T14:57:06,639 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36531 is added to blk_1073741854_1030 (size=110) 2024-11-16T14:57:06,640 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40641 is added to blk_1073741854_1030 (size=110) 2024-11-16T14:57:06,641 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(898): Instantiated testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,qqq,1731769026392.18f03e2509a3a89e4a69ba87fad73940.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-16T14:57:06,641 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(1722): Closing 18f03e2509a3a89e4a69ba87fad73940, disabling compactions & flushes 2024-11-16T14:57:06,641 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(1755): Closing region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,qqq,1731769026392.18f03e2509a3a89e4a69ba87fad73940. 2024-11-16T14:57:06,641 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,qqq,1731769026392.18f03e2509a3a89e4a69ba87fad73940. 2024-11-16T14:57:06,641 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,qqq,1731769026392.18f03e2509a3a89e4a69ba87fad73940. after waiting 0 ms 2024-11-16T14:57:06,641 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,qqq,1731769026392.18f03e2509a3a89e4a69ba87fad73940. 2024-11-16T14:57:06,641 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(1973): Closed testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,qqq,1731769026392.18f03e2509a3a89e4a69ba87fad73940. 2024-11-16T14:57:06,641 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(1676): Region close journal for 18f03e2509a3a89e4a69ba87fad73940: Waiting for close lock at 1731769026641Disabling compacts and flushes for region at 1731769026641Disabling writes for close at 1731769026641Writing region close event to WAL at 1731769026641Closed at 1731769026641 2024-11-16T14:57:06,642 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => b364a006589fe0d2474ffa45692a6efc, NAME => 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,rrr,1731769026392.b364a006589fe0d2474ffa45692a6efc.', STARTKEY => 'rrr', ENDKEY => 'sss'}, tableDescriptor='testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'colfamily11', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'colfamily21', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'colfamily31', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:42223/user/jenkins/test-data/a8176cb1-2ebb-b530-b3c7-53c9e53c186e 2024-11-16T14:57:06,657 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40641 is added to blk_1073741855_1031 (size=110) 2024-11-16T14:57:06,657 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36531 is added to blk_1073741855_1031 (size=110) 2024-11-16T14:57:06,657 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41671 is added to blk_1073741855_1031 (size=110) 2024-11-16T14:57:06,658 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(898): Instantiated testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,rrr,1731769026392.b364a006589fe0d2474ffa45692a6efc.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-16T14:57:06,658 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(1722): Closing b364a006589fe0d2474ffa45692a6efc, disabling compactions & flushes 2024-11-16T14:57:06,658 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(1755): Closing region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,rrr,1731769026392.b364a006589fe0d2474ffa45692a6efc. 2024-11-16T14:57:06,658 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,rrr,1731769026392.b364a006589fe0d2474ffa45692a6efc. 2024-11-16T14:57:06,658 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,rrr,1731769026392.b364a006589fe0d2474ffa45692a6efc. after waiting 0 ms 2024-11-16T14:57:06,658 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,rrr,1731769026392.b364a006589fe0d2474ffa45692a6efc. 2024-11-16T14:57:06,658 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(1973): Closed testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,rrr,1731769026392.b364a006589fe0d2474ffa45692a6efc. 2024-11-16T14:57:06,659 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(1676): Region close journal for b364a006589fe0d2474ffa45692a6efc: Waiting for close lock at 1731769026658Disabling compacts and flushes for region at 1731769026658Disabling writes for close at 1731769026658Writing region close event to WAL at 1731769026658Closed at 1731769026658 2024-11-16T14:57:06,659 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 33054df03c1be51dc860405880be6075, NAME => 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,sss,1731769026392.33054df03c1be51dc860405880be6075.', STARTKEY => 'sss', ENDKEY => 'ttt'}, tableDescriptor='testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'colfamily11', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'colfamily21', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'colfamily31', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:42223/user/jenkins/test-data/a8176cb1-2ebb-b530-b3c7-53c9e53c186e 2024-11-16T14:57:06,669 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41671 is added to blk_1073741856_1032 (size=110) 2024-11-16T14:57:06,669 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40641 is added to blk_1073741856_1032 (size=110) 2024-11-16T14:57:06,670 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36531 is added to blk_1073741856_1032 (size=110) 2024-11-16T14:57:06,670 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(898): Instantiated testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,sss,1731769026392.33054df03c1be51dc860405880be6075.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-16T14:57:06,670 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(1722): Closing 33054df03c1be51dc860405880be6075, disabling compactions & flushes 2024-11-16T14:57:06,670 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(1755): Closing region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,sss,1731769026392.33054df03c1be51dc860405880be6075. 2024-11-16T14:57:06,670 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,sss,1731769026392.33054df03c1be51dc860405880be6075. 2024-11-16T14:57:06,670 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,sss,1731769026392.33054df03c1be51dc860405880be6075. after waiting 0 ms 2024-11-16T14:57:06,671 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,sss,1731769026392.33054df03c1be51dc860405880be6075. 2024-11-16T14:57:06,671 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(1973): Closed testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,sss,1731769026392.33054df03c1be51dc860405880be6075. 2024-11-16T14:57:06,671 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(1676): Region close journal for 33054df03c1be51dc860405880be6075: Waiting for close lock at 1731769026670Disabling compacts and flushes for region at 1731769026670Disabling writes for close at 1731769026671 (+1 ms)Writing region close event to WAL at 1731769026671Closed at 1731769026671 2024-11-16T14:57:06,671 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 7f11f79823720d664105d8643dfa862d, NAME => 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ttt,1731769026392.7f11f79823720d664105d8643dfa862d.', STARTKEY => 'ttt', ENDKEY => 'uuu'}, tableDescriptor='testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'colfamily11', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'colfamily21', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'colfamily31', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:42223/user/jenkins/test-data/a8176cb1-2ebb-b530-b3c7-53c9e53c186e 2024-11-16T14:57:06,684 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41671 is added to blk_1073741857_1033 (size=110) 2024-11-16T14:57:06,684 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36531 is added to blk_1073741857_1033 (size=110) 2024-11-16T14:57:06,684 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40641 is added to blk_1073741857_1033 (size=110) 2024-11-16T14:57:06,685 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(898): Instantiated testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ttt,1731769026392.7f11f79823720d664105d8643dfa862d.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-16T14:57:06,685 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(1722): Closing 7f11f79823720d664105d8643dfa862d, disabling compactions & flushes 2024-11-16T14:57:06,685 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(1755): Closing region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ttt,1731769026392.7f11f79823720d664105d8643dfa862d. 2024-11-16T14:57:06,685 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ttt,1731769026392.7f11f79823720d664105d8643dfa862d. 2024-11-16T14:57:06,685 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ttt,1731769026392.7f11f79823720d664105d8643dfa862d. after waiting 0 ms 2024-11-16T14:57:06,685 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ttt,1731769026392.7f11f79823720d664105d8643dfa862d. 2024-11-16T14:57:06,685 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(1973): Closed testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ttt,1731769026392.7f11f79823720d664105d8643dfa862d. 2024-11-16T14:57:06,685 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(1676): Region close journal for 7f11f79823720d664105d8643dfa862d: Waiting for close lock at 1731769026685Disabling compacts and flushes for region at 1731769026685Disabling writes for close at 1731769026685Writing region close event to WAL at 1731769026685Closed at 1731769026685 2024-11-16T14:57:06,686 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 35ef9a7f35bb0b08ec1d53b82d3f9777, NAME => 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,uuu,1731769026392.35ef9a7f35bb0b08ec1d53b82d3f9777.', STARTKEY => 'uuu', ENDKEY => 'vvv'}, tableDescriptor='testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'colfamily11', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'colfamily21', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'colfamily31', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:42223/user/jenkins/test-data/a8176cb1-2ebb-b530-b3c7-53c9e53c186e 2024-11-16T14:57:06,695 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41671 is added to blk_1073741858_1034 (size=110) 2024-11-16T14:57:06,696 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40641 is added to blk_1073741858_1034 (size=110) 2024-11-16T14:57:06,696 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36531 is added to blk_1073741858_1034 (size=110) 2024-11-16T14:57:06,697 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(898): Instantiated testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,uuu,1731769026392.35ef9a7f35bb0b08ec1d53b82d3f9777.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-16T14:57:06,697 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(1722): Closing 35ef9a7f35bb0b08ec1d53b82d3f9777, disabling compactions & flushes 2024-11-16T14:57:06,697 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(1755): Closing region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,uuu,1731769026392.35ef9a7f35bb0b08ec1d53b82d3f9777. 2024-11-16T14:57:06,697 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,uuu,1731769026392.35ef9a7f35bb0b08ec1d53b82d3f9777. 2024-11-16T14:57:06,697 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,uuu,1731769026392.35ef9a7f35bb0b08ec1d53b82d3f9777. after waiting 0 ms 2024-11-16T14:57:06,697 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,uuu,1731769026392.35ef9a7f35bb0b08ec1d53b82d3f9777. 2024-11-16T14:57:06,697 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(1973): Closed testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,uuu,1731769026392.35ef9a7f35bb0b08ec1d53b82d3f9777. 2024-11-16T14:57:06,697 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(1676): Region close journal for 35ef9a7f35bb0b08ec1d53b82d3f9777: Waiting for close lock at 1731769026697Disabling compacts and flushes for region at 1731769026697Disabling writes for close at 1731769026697Writing region close event to WAL at 1731769026697Closed at 1731769026697 2024-11-16T14:57:06,698 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => b5566c7e10a0e001c7f58316bdd43c2f, NAME => 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,vvv,1731769026392.b5566c7e10a0e001c7f58316bdd43c2f.', STARTKEY => 'vvv', ENDKEY => 'www'}, tableDescriptor='testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'colfamily11', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'colfamily21', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'colfamily31', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:42223/user/jenkins/test-data/a8176cb1-2ebb-b530-b3c7-53c9e53c186e 2024-11-16T14:57:06,708 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40641 is added to blk_1073741859_1035 (size=110) 2024-11-16T14:57:06,708 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41671 is added to blk_1073741859_1035 (size=110) 2024-11-16T14:57:06,708 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36531 is added to blk_1073741859_1035 (size=110) 2024-11-16T14:57:06,709 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(898): Instantiated testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,vvv,1731769026392.b5566c7e10a0e001c7f58316bdd43c2f.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-16T14:57:06,709 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(1722): Closing b5566c7e10a0e001c7f58316bdd43c2f, disabling compactions & flushes 2024-11-16T14:57:06,709 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(1755): Closing region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,vvv,1731769026392.b5566c7e10a0e001c7f58316bdd43c2f. 2024-11-16T14:57:06,709 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,vvv,1731769026392.b5566c7e10a0e001c7f58316bdd43c2f. 2024-11-16T14:57:06,709 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,vvv,1731769026392.b5566c7e10a0e001c7f58316bdd43c2f. after waiting 0 ms 2024-11-16T14:57:06,709 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,vvv,1731769026392.b5566c7e10a0e001c7f58316bdd43c2f. 2024-11-16T14:57:06,709 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(1973): Closed testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,vvv,1731769026392.b5566c7e10a0e001c7f58316bdd43c2f. 2024-11-16T14:57:06,709 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(1676): Region close journal for b5566c7e10a0e001c7f58316bdd43c2f: Waiting for close lock at 1731769026709Disabling compacts and flushes for region at 1731769026709Disabling writes for close at 1731769026709Writing region close event to WAL at 1731769026709Closed at 1731769026709 2024-11-16T14:57:06,710 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 62a6d124ad9d161782c0632102c766de, NAME => 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,www,1731769026392.62a6d124ad9d161782c0632102c766de.', STARTKEY => 'www', ENDKEY => 'xxx'}, tableDescriptor='testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'colfamily11', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'colfamily21', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'colfamily31', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:42223/user/jenkins/test-data/a8176cb1-2ebb-b530-b3c7-53c9e53c186e 2024-11-16T14:57:06,719 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41671 is added to blk_1073741860_1036 (size=110) 2024-11-16T14:57:06,720 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36531 is added to blk_1073741860_1036 (size=110) 2024-11-16T14:57:06,721 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40641 is added to blk_1073741860_1036 (size=110) 2024-11-16T14:57:06,721 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(898): Instantiated testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,www,1731769026392.62a6d124ad9d161782c0632102c766de.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-16T14:57:06,721 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(1722): Closing 62a6d124ad9d161782c0632102c766de, disabling compactions & flushes 2024-11-16T14:57:06,722 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(1755): Closing region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,www,1731769026392.62a6d124ad9d161782c0632102c766de. 2024-11-16T14:57:06,722 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,www,1731769026392.62a6d124ad9d161782c0632102c766de. 2024-11-16T14:57:06,722 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,www,1731769026392.62a6d124ad9d161782c0632102c766de. after waiting 0 ms 2024-11-16T14:57:06,722 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,www,1731769026392.62a6d124ad9d161782c0632102c766de. 2024-11-16T14:57:06,722 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(1973): Closed testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,www,1731769026392.62a6d124ad9d161782c0632102c766de. 2024-11-16T14:57:06,722 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(1676): Region close journal for 62a6d124ad9d161782c0632102c766de: Waiting for close lock at 1731769026721Disabling compacts and flushes for region at 1731769026721Disabling writes for close at 1731769026722 (+1 ms)Writing region close event to WAL at 1731769026722Closed at 1731769026722 2024-11-16T14:57:06,723 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => dc4fcaece350cd66f6bf88cfbdff6f15, NAME => 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,xxx,1731769026392.dc4fcaece350cd66f6bf88cfbdff6f15.', STARTKEY => 'xxx', ENDKEY => 'yyy'}, tableDescriptor='testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'colfamily11', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'colfamily21', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'colfamily31', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:42223/user/jenkins/test-data/a8176cb1-2ebb-b530-b3c7-53c9e53c186e 2024-11-16T14:57:06,733 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36531 is added to blk_1073741861_1037 (size=110) 2024-11-16T14:57:06,733 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40641 is added to blk_1073741861_1037 (size=110) 2024-11-16T14:57:06,734 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41671 is added to blk_1073741861_1037 (size=110) 2024-11-16T14:57:06,734 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(898): Instantiated testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,xxx,1731769026392.dc4fcaece350cd66f6bf88cfbdff6f15.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-16T14:57:06,735 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(1722): Closing dc4fcaece350cd66f6bf88cfbdff6f15, disabling compactions & flushes 2024-11-16T14:57:06,735 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(1755): Closing region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,xxx,1731769026392.dc4fcaece350cd66f6bf88cfbdff6f15. 2024-11-16T14:57:06,735 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,xxx,1731769026392.dc4fcaece350cd66f6bf88cfbdff6f15. 2024-11-16T14:57:06,735 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,xxx,1731769026392.dc4fcaece350cd66f6bf88cfbdff6f15. after waiting 0 ms 2024-11-16T14:57:06,735 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,xxx,1731769026392.dc4fcaece350cd66f6bf88cfbdff6f15. 2024-11-16T14:57:06,735 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(1973): Closed testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,xxx,1731769026392.dc4fcaece350cd66f6bf88cfbdff6f15. 2024-11-16T14:57:06,735 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(1676): Region close journal for dc4fcaece350cd66f6bf88cfbdff6f15: Waiting for close lock at 1731769026735Disabling compacts and flushes for region at 1731769026735Disabling writes for close at 1731769026735Writing region close event to WAL at 1731769026735Closed at 1731769026735 2024-11-16T14:57:06,735 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 426ff231e487ce9f8492776ca1dacc85, NAME => 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,yyy,1731769026392.426ff231e487ce9f8492776ca1dacc85.', STARTKEY => 'yyy', ENDKEY => 'zzz'}, tableDescriptor='testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'colfamily11', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'colfamily21', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'colfamily31', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:42223/user/jenkins/test-data/a8176cb1-2ebb-b530-b3c7-53c9e53c186e 2024-11-16T14:57:06,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43615 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-16T14:57:06,744 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36531 is added to blk_1073741862_1038 (size=110) 2024-11-16T14:57:06,744 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41671 is added to blk_1073741862_1038 (size=110) 2024-11-16T14:57:06,745 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40641 is added to blk_1073741862_1038 (size=110) 2024-11-16T14:57:06,746 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(898): Instantiated testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,yyy,1731769026392.426ff231e487ce9f8492776ca1dacc85.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-16T14:57:06,746 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(1722): Closing 426ff231e487ce9f8492776ca1dacc85, disabling compactions & flushes 2024-11-16T14:57:06,746 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(1755): Closing region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,yyy,1731769026392.426ff231e487ce9f8492776ca1dacc85. 2024-11-16T14:57:06,746 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,yyy,1731769026392.426ff231e487ce9f8492776ca1dacc85. 2024-11-16T14:57:06,746 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,yyy,1731769026392.426ff231e487ce9f8492776ca1dacc85. after waiting 0 ms 2024-11-16T14:57:06,746 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,yyy,1731769026392.426ff231e487ce9f8492776ca1dacc85. 2024-11-16T14:57:06,746 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(1973): Closed testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,yyy,1731769026392.426ff231e487ce9f8492776ca1dacc85. 2024-11-16T14:57:06,746 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(1676): Region close journal for 426ff231e487ce9f8492776ca1dacc85: Waiting for close lock at 1731769026746Disabling compacts and flushes for region at 1731769026746Disabling writes for close at 1731769026746Writing region close event to WAL at 1731769026746Closed at 1731769026746 2024-11-16T14:57:06,747 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 2f41a8c996224c818e917389bd3f5141, NAME => 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,zzz,1731769026392.2f41a8c996224c818e917389bd3f5141.', STARTKEY => 'zzz', ENDKEY => ''}, tableDescriptor='testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'colfamily11', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'colfamily21', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'colfamily31', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:42223/user/jenkins/test-data/a8176cb1-2ebb-b530-b3c7-53c9e53c186e 2024-11-16T14:57:06,757 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41671 is added to blk_1073741863_1039 (size=107) 2024-11-16T14:57:06,758 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40641 is added to blk_1073741863_1039 (size=107) 2024-11-16T14:57:06,758 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36531 is added to blk_1073741863_1039 (size=107) 2024-11-16T14:57:06,759 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(898): Instantiated testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,zzz,1731769026392.2f41a8c996224c818e917389bd3f5141.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-16T14:57:06,759 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(1722): Closing 2f41a8c996224c818e917389bd3f5141, disabling compactions & flushes 2024-11-16T14:57:06,759 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(1755): Closing region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,zzz,1731769026392.2f41a8c996224c818e917389bd3f5141. 2024-11-16T14:57:06,759 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,zzz,1731769026392.2f41a8c996224c818e917389bd3f5141. 2024-11-16T14:57:06,759 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,zzz,1731769026392.2f41a8c996224c818e917389bd3f5141. after waiting 0 ms 2024-11-16T14:57:06,759 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,zzz,1731769026392.2f41a8c996224c818e917389bd3f5141. 2024-11-16T14:57:06,759 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(1973): Closed testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,zzz,1731769026392.2f41a8c996224c818e917389bd3f5141. 2024-11-16T14:57:06,759 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(1676): Region close journal for 2f41a8c996224c818e917389bd3f5141: Waiting for close lock at 1731769026759Disabling compacts and flushes for region at 1731769026759Disabling writes for close at 1731769026759Writing region close event to WAL at 1731769026759Closed at 1731769026759 2024-11-16T14:57:06,909 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-2 {}] regionserver.HRegion(898): Instantiated testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ggg,1731769026392.dcd6414ddebad66e87ab37a6dba924f3.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-16T14:57:06,909 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-2 {}] regionserver.HRegion(1722): Closing dcd6414ddebad66e87ab37a6dba924f3, disabling compactions & flushes 2024-11-16T14:57:06,909 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-2 {}] regionserver.HRegion(1755): Closing region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ggg,1731769026392.dcd6414ddebad66e87ab37a6dba924f3. 2024-11-16T14:57:06,909 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-2 {}] regionserver.HRegion(1776): Time limited wait for close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ggg,1731769026392.dcd6414ddebad66e87ab37a6dba924f3. 2024-11-16T14:57:06,909 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-2 {}] regionserver.HRegion(1843): Acquired close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ggg,1731769026392.dcd6414ddebad66e87ab37a6dba924f3. after waiting 0 ms 2024-11-16T14:57:06,909 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-2 {}] regionserver.HRegion(1853): Updates disabled for region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ggg,1731769026392.dcd6414ddebad66e87ab37a6dba924f3. 2024-11-16T14:57:06,910 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-2 {}] regionserver.HRegion(1973): Closed testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ggg,1731769026392.dcd6414ddebad66e87ab37a6dba924f3. 2024-11-16T14:57:06,910 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-2 {}] regionserver.HRegion(1676): Region close journal for dcd6414ddebad66e87ab37a6dba924f3: Waiting for close lock at 1731769026909Disabling compacts and flushes for region at 1731769026909Disabling writes for close at 1731769026909Writing region close event to WAL at 1731769026909Closed at 1731769026909 2024-11-16T14:57:06,934 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-1 {}] regionserver.HRegion(898): Instantiated testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,iii,1731769026392.7352dcdbf9811dcebc51a714a1de4f55.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-16T14:57:06,934 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-1 {}] regionserver.HRegion(1722): Closing 7352dcdbf9811dcebc51a714a1de4f55, disabling compactions & flushes 2024-11-16T14:57:06,934 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-1 {}] regionserver.HRegion(1755): Closing region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,iii,1731769026392.7352dcdbf9811dcebc51a714a1de4f55. 2024-11-16T14:57:06,934 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,iii,1731769026392.7352dcdbf9811dcebc51a714a1de4f55. 2024-11-16T14:57:06,934 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-1 {}] regionserver.HRegion(1843): Acquired close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,iii,1731769026392.7352dcdbf9811dcebc51a714a1de4f55. after waiting 0 ms 2024-11-16T14:57:06,934 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-1 {}] regionserver.HRegion(1853): Updates disabled for region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,iii,1731769026392.7352dcdbf9811dcebc51a714a1de4f55. 2024-11-16T14:57:06,934 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-1 {}] regionserver.HRegion(1973): Closed testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,iii,1731769026392.7352dcdbf9811dcebc51a714a1de4f55. 2024-11-16T14:57:06,935 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-1 {}] regionserver.HRegion(1676): Region close journal for 7352dcdbf9811dcebc51a714a1de4f55: Waiting for close lock at 1731769026934Disabling compacts and flushes for region at 1731769026934Disabling writes for close at 1731769026934Writing region close event to WAL at 1731769026934Closed at 1731769026934 2024-11-16T14:57:06,938 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true execute state=CREATE_TABLE_ADD_TO_META 2024-11-16T14:57:06,944 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,bbb,1731769026392.c3dde5f5f668d914608a2987d4f4c001.","families":{"info":[{"qualifier":"regioninfo","vlen":109,"tag":[],"timestamp":"1731769026938"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731769026938"}]},"ts":"1731769026938"} 2024-11-16T14:57:06,944 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,,1731769026392.4f0d635e77a1e3dd04c4d107bb0f1564.","families":{"info":[{"qualifier":"regioninfo","vlen":106,"tag":[],"timestamp":"1731769026938"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731769026938"}]},"ts":"1731769026938"} 2024-11-16T14:57:06,944 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ccc,1731769026392.5bb22144956dcd43c9b1d957232b11e7.","families":{"info":[{"qualifier":"regioninfo","vlen":109,"tag":[],"timestamp":"1731769026938"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731769026938"}]},"ts":"1731769026938"} 2024-11-16T14:57:06,944 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,fff,1731769026392.a5db9431e22152bd2a82e84504d2879a.","families":{"info":[{"qualifier":"regioninfo","vlen":109,"tag":[],"timestamp":"1731769026938"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731769026938"}]},"ts":"1731769026938"} 2024-11-16T14:57:06,944 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,eee,1731769026392.fd3944fc2b78246a853fb2232c747c35.","families":{"info":[{"qualifier":"regioninfo","vlen":109,"tag":[],"timestamp":"1731769026938"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731769026938"}]},"ts":"1731769026938"} 2024-11-16T14:57:06,944 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ddd,1731769026392.dde7f54607f11c5fb4271e361e82dabd.","families":{"info":[{"qualifier":"regioninfo","vlen":109,"tag":[],"timestamp":"1731769026938"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731769026938"}]},"ts":"1731769026938"} 2024-11-16T14:57:06,945 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,hhh,1731769026392.1d97df67e5110b0fe1a1e4b7c1341cbb.","families":{"info":[{"qualifier":"regioninfo","vlen":109,"tag":[],"timestamp":"1731769026938"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731769026938"}]},"ts":"1731769026938"} 2024-11-16T14:57:06,945 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,jjj,1731769026392.7eb7b75d28b808c5c3f2f96e407b4347.","families":{"info":[{"qualifier":"regioninfo","vlen":109,"tag":[],"timestamp":"1731769026938"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731769026938"}]},"ts":"1731769026938"} 2024-11-16T14:57:06,945 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,kkk,1731769026392.b4f73326a2a6a279d503e557c6ed452f.","families":{"info":[{"qualifier":"regioninfo","vlen":109,"tag":[],"timestamp":"1731769026938"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731769026938"}]},"ts":"1731769026938"} 2024-11-16T14:57:06,945 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,lll,1731769026392.266565e60d7419a2a30ceec1b0f6519b.","families":{"info":[{"qualifier":"regioninfo","vlen":109,"tag":[],"timestamp":"1731769026938"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731769026938"}]},"ts":"1731769026938"} 2024-11-16T14:57:06,945 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,mmm,1731769026392.c2acbc05ae7bb1607a5719b701b40383.","families":{"info":[{"qualifier":"regioninfo","vlen":109,"tag":[],"timestamp":"1731769026938"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731769026938"}]},"ts":"1731769026938"} 2024-11-16T14:57:06,945 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,nnn,1731769026392.994ca1526cf6c4838e65f85444853f1a.","families":{"info":[{"qualifier":"regioninfo","vlen":109,"tag":[],"timestamp":"1731769026938"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731769026938"}]},"ts":"1731769026938"} 2024-11-16T14:57:06,946 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ooo,1731769026392.299131947f15f9c44bcfe22cd6623096.","families":{"info":[{"qualifier":"regioninfo","vlen":109,"tag":[],"timestamp":"1731769026938"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731769026938"}]},"ts":"1731769026938"} 2024-11-16T14:57:06,946 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ppp,1731769026392.7f5c9df3a544933227f58b052f743f72.","families":{"info":[{"qualifier":"regioninfo","vlen":109,"tag":[],"timestamp":"1731769026938"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731769026938"}]},"ts":"1731769026938"} 2024-11-16T14:57:06,946 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,qqq,1731769026392.18f03e2509a3a89e4a69ba87fad73940.","families":{"info":[{"qualifier":"regioninfo","vlen":109,"tag":[],"timestamp":"1731769026938"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731769026938"}]},"ts":"1731769026938"} 2024-11-16T14:57:06,946 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,rrr,1731769026392.b364a006589fe0d2474ffa45692a6efc.","families":{"info":[{"qualifier":"regioninfo","vlen":109,"tag":[],"timestamp":"1731769026938"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731769026938"}]},"ts":"1731769026938"} 2024-11-16T14:57:06,946 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,sss,1731769026392.33054df03c1be51dc860405880be6075.","families":{"info":[{"qualifier":"regioninfo","vlen":109,"tag":[],"timestamp":"1731769026938"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731769026938"}]},"ts":"1731769026938"} 2024-11-16T14:57:06,946 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ttt,1731769026392.7f11f79823720d664105d8643dfa862d.","families":{"info":[{"qualifier":"regioninfo","vlen":109,"tag":[],"timestamp":"1731769026938"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731769026938"}]},"ts":"1731769026938"} 2024-11-16T14:57:06,946 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,uuu,1731769026392.35ef9a7f35bb0b08ec1d53b82d3f9777.","families":{"info":[{"qualifier":"regioninfo","vlen":109,"tag":[],"timestamp":"1731769026938"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731769026938"}]},"ts":"1731769026938"} 2024-11-16T14:57:06,947 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,vvv,1731769026392.b5566c7e10a0e001c7f58316bdd43c2f.","families":{"info":[{"qualifier":"regioninfo","vlen":109,"tag":[],"timestamp":"1731769026938"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731769026938"}]},"ts":"1731769026938"} 2024-11-16T14:57:06,947 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,www,1731769026392.62a6d124ad9d161782c0632102c766de.","families":{"info":[{"qualifier":"regioninfo","vlen":109,"tag":[],"timestamp":"1731769026938"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731769026938"}]},"ts":"1731769026938"} 2024-11-16T14:57:06,947 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,xxx,1731769026392.dc4fcaece350cd66f6bf88cfbdff6f15.","families":{"info":[{"qualifier":"regioninfo","vlen":109,"tag":[],"timestamp":"1731769026938"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731769026938"}]},"ts":"1731769026938"} 2024-11-16T14:57:06,947 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,yyy,1731769026392.426ff231e487ce9f8492776ca1dacc85.","families":{"info":[{"qualifier":"regioninfo","vlen":109,"tag":[],"timestamp":"1731769026938"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731769026938"}]},"ts":"1731769026938"} 2024-11-16T14:57:06,947 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,zzz,1731769026392.2f41a8c996224c818e917389bd3f5141.","families":{"info":[{"qualifier":"regioninfo","vlen":106,"tag":[],"timestamp":"1731769026938"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731769026938"}]},"ts":"1731769026938"} 2024-11-16T14:57:06,947 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ggg,1731769026392.dcd6414ddebad66e87ab37a6dba924f3.","families":{"info":[{"qualifier":"regioninfo","vlen":109,"tag":[],"timestamp":"1731769026938"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731769026938"}]},"ts":"1731769026938"} 2024-11-16T14:57:06,947 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,iii,1731769026392.7352dcdbf9811dcebc51a714a1de4f55.","families":{"info":[{"qualifier":"regioninfo","vlen":109,"tag":[],"timestamp":"1731769026938"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731769026938"}]},"ts":"1731769026938"} 2024-11-16T14:57:06,989 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(832): Added 26 regions to meta. 2024-11-16T14:57:06,991 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-16T14:57:06,993 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731769026991"}]},"ts":"1731769026991"} 2024-11-16T14:57:06,998 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, state=ENABLING in hbase:meta 2024-11-16T14:57:06,999 DEBUG [PEWorker-1 {}] balancer.BalancerClusterState(204): Hosts are {fc3fda29f660=0} racks are {/default-rack=0} 2024-11-16T14:57:07,036 DEBUG [PEWorker-1 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-11-16T14:57:07,036 DEBUG [PEWorker-1 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-11-16T14:57:07,036 DEBUG [PEWorker-1 {}] balancer.BalancerClusterState(303): server 2 has 0 regions 2024-11-16T14:57:07,037 DEBUG [PEWorker-1 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-11-16T14:57:07,037 DEBUG [PEWorker-1 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-11-16T14:57:07,037 DEBUG [PEWorker-1 {}] balancer.BalancerClusterState(310): server 2 is on host 0 2024-11-16T14:57:07,037 INFO [PEWorker-1 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-11-16T14:57:07,037 INFO [PEWorker-1 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-11-16T14:57:07,037 INFO [PEWorker-1 {}] balancer.BalancerClusterState(321): server 2 is on rack 0 2024-11-16T14:57:07,037 DEBUG [PEWorker-1 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-11-16T14:57:07,039 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=4f0d635e77a1e3dd04c4d107bb0f1564, ASSIGN}, {pid=6, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=c3dde5f5f668d914608a2987d4f4c001, ASSIGN}, {pid=7, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=5bb22144956dcd43c9b1d957232b11e7, ASSIGN}, {pid=8, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=dde7f54607f11c5fb4271e361e82dabd, ASSIGN}, {pid=9, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=fd3944fc2b78246a853fb2232c747c35, ASSIGN}, {pid=10, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=a5db9431e22152bd2a82e84504d2879a, ASSIGN}, {pid=11, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=dcd6414ddebad66e87ab37a6dba924f3, ASSIGN}, {pid=12, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=1d97df67e5110b0fe1a1e4b7c1341cbb, ASSIGN}, {pid=13, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=7352dcdbf9811dcebc51a714a1de4f55, ASSIGN}, {pid=14, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=7eb7b75d28b808c5c3f2f96e407b4347, ASSIGN}, {pid=15, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=b4f73326a2a6a279d503e557c6ed452f, ASSIGN}, {pid=16, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=266565e60d7419a2a30ceec1b0f6519b, ASSIGN}, {pid=17, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=c2acbc05ae7bb1607a5719b701b40383, ASSIGN}, {pid=18, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=994ca1526cf6c4838e65f85444853f1a, ASSIGN}, {pid=19, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=299131947f15f9c44bcfe22cd6623096, ASSIGN}, {pid=20, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=7f5c9df3a544933227f58b052f743f72, ASSIGN}, {pid=21, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=18f03e2509a3a89e4a69ba87fad73940, ASSIGN}, {pid=22, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=b364a006589fe0d2474ffa45692a6efc, ASSIGN}, {pid=23, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=33054df03c1be51dc860405880be6075, ASSIGN}, {pid=24, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=7f11f79823720d664105d8643dfa862d, ASSIGN}, {pid=25, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=35ef9a7f35bb0b08ec1d53b82d3f9777, ASSIGN}, {pid=26, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=b5566c7e10a0e001c7f58316bdd43c2f, ASSIGN}, {pid=27, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=62a6d124ad9d161782c0632102c766de, ASSIGN}, {pid=28, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=dc4fcaece350cd66f6bf88cfbdff6f15, ASSIGN}, {pid=29, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=426ff231e487ce9f8492776ca1dacc85, ASSIGN}, {pid=30, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=2f41a8c996224c818e917389bd3f5141, ASSIGN}] 2024-11-16T14:57:07,049 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=30, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=2f41a8c996224c818e917389bd3f5141, ASSIGN 2024-11-16T14:57:07,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43615 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-16T14:57:07,051 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=30, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=2f41a8c996224c818e917389bd3f5141, ASSIGN; state=OFFLINE, location=fc3fda29f660,39089,1731769023769; forceNewPlan=false, retain=false 2024-11-16T14:57:07,052 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=29, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=426ff231e487ce9f8492776ca1dacc85, ASSIGN 2024-11-16T14:57:07,054 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=29, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=426ff231e487ce9f8492776ca1dacc85, ASSIGN; state=OFFLINE, location=fc3fda29f660,38897,1731769023879; forceNewPlan=false, retain=false 2024-11-16T14:57:07,055 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=28, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=dc4fcaece350cd66f6bf88cfbdff6f15, ASSIGN 2024-11-16T14:57:07,056 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=28, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=dc4fcaece350cd66f6bf88cfbdff6f15, ASSIGN; state=OFFLINE, location=fc3fda29f660,42985,1731769023951; forceNewPlan=false, retain=false 2024-11-16T14:57:07,058 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=27, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=62a6d124ad9d161782c0632102c766de, ASSIGN 2024-11-16T14:57:07,060 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=27, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=62a6d124ad9d161782c0632102c766de, ASSIGN; state=OFFLINE, location=fc3fda29f660,39089,1731769023769; forceNewPlan=false, retain=false 2024-11-16T14:57:07,061 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=26, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=b5566c7e10a0e001c7f58316bdd43c2f, ASSIGN 2024-11-16T14:57:07,063 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=26, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=b5566c7e10a0e001c7f58316bdd43c2f, ASSIGN; state=OFFLINE, location=fc3fda29f660,38897,1731769023879; forceNewPlan=false, retain=false 2024-11-16T14:57:07,064 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=25, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=35ef9a7f35bb0b08ec1d53b82d3f9777, ASSIGN 2024-11-16T14:57:07,066 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=25, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=35ef9a7f35bb0b08ec1d53b82d3f9777, ASSIGN; state=OFFLINE, location=fc3fda29f660,42985,1731769023951; forceNewPlan=false, retain=false 2024-11-16T14:57:07,067 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=24, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=7f11f79823720d664105d8643dfa862d, ASSIGN 2024-11-16T14:57:07,068 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=24, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=7f11f79823720d664105d8643dfa862d, ASSIGN; state=OFFLINE, location=fc3fda29f660,39089,1731769023769; forceNewPlan=false, retain=false 2024-11-16T14:57:07,069 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=23, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=33054df03c1be51dc860405880be6075, ASSIGN 2024-11-16T14:57:07,071 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=23, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=33054df03c1be51dc860405880be6075, ASSIGN; state=OFFLINE, location=fc3fda29f660,38897,1731769023879; forceNewPlan=false, retain=false 2024-11-16T14:57:07,072 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=22, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=b364a006589fe0d2474ffa45692a6efc, ASSIGN 2024-11-16T14:57:07,074 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=22, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=b364a006589fe0d2474ffa45692a6efc, ASSIGN; state=OFFLINE, location=fc3fda29f660,42985,1731769023951; forceNewPlan=false, retain=false 2024-11-16T14:57:07,075 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=21, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=18f03e2509a3a89e4a69ba87fad73940, ASSIGN 2024-11-16T14:57:07,077 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=21, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=18f03e2509a3a89e4a69ba87fad73940, ASSIGN; state=OFFLINE, location=fc3fda29f660,39089,1731769023769; forceNewPlan=false, retain=false 2024-11-16T14:57:07,079 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=20, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=7f5c9df3a544933227f58b052f743f72, ASSIGN 2024-11-16T14:57:07,080 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=20, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=7f5c9df3a544933227f58b052f743f72, ASSIGN; state=OFFLINE, location=fc3fda29f660,38897,1731769023879; forceNewPlan=false, retain=false 2024-11-16T14:57:07,081 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=19, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=299131947f15f9c44bcfe22cd6623096, ASSIGN 2024-11-16T14:57:07,083 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=19, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=299131947f15f9c44bcfe22cd6623096, ASSIGN; state=OFFLINE, location=fc3fda29f660,42985,1731769023951; forceNewPlan=false, retain=false 2024-11-16T14:57:07,084 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=18, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=994ca1526cf6c4838e65f85444853f1a, ASSIGN 2024-11-16T14:57:07,086 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=18, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=994ca1526cf6c4838e65f85444853f1a, ASSIGN; state=OFFLINE, location=fc3fda29f660,39089,1731769023769; forceNewPlan=false, retain=false 2024-11-16T14:57:07,088 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=17, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=c2acbc05ae7bb1607a5719b701b40383, ASSIGN 2024-11-16T14:57:07,089 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=17, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=c2acbc05ae7bb1607a5719b701b40383, ASSIGN; state=OFFLINE, location=fc3fda29f660,38897,1731769023879; forceNewPlan=false, retain=false 2024-11-16T14:57:07,091 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=16, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=266565e60d7419a2a30ceec1b0f6519b, ASSIGN 2024-11-16T14:57:07,093 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=16, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=266565e60d7419a2a30ceec1b0f6519b, ASSIGN; state=OFFLINE, location=fc3fda29f660,42985,1731769023951; forceNewPlan=false, retain=false 2024-11-16T14:57:07,094 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=15, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=b4f73326a2a6a279d503e557c6ed452f, ASSIGN 2024-11-16T14:57:07,095 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=15, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=b4f73326a2a6a279d503e557c6ed452f, ASSIGN; state=OFFLINE, location=fc3fda29f660,39089,1731769023769; forceNewPlan=false, retain=false 2024-11-16T14:57:07,097 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=14, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=7eb7b75d28b808c5c3f2f96e407b4347, ASSIGN 2024-11-16T14:57:07,098 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=14, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=7eb7b75d28b808c5c3f2f96e407b4347, ASSIGN; state=OFFLINE, location=fc3fda29f660,38897,1731769023879; forceNewPlan=false, retain=false 2024-11-16T14:57:07,099 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=13, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=7352dcdbf9811dcebc51a714a1de4f55, ASSIGN 2024-11-16T14:57:07,100 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=13, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=7352dcdbf9811dcebc51a714a1de4f55, ASSIGN; state=OFFLINE, location=fc3fda29f660,38897,1731769023879; forceNewPlan=false, retain=false 2024-11-16T14:57:07,102 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=12, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=1d97df67e5110b0fe1a1e4b7c1341cbb, ASSIGN 2024-11-16T14:57:07,103 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=12, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=1d97df67e5110b0fe1a1e4b7c1341cbb, ASSIGN; state=OFFLINE, location=fc3fda29f660,42985,1731769023951; forceNewPlan=false, retain=false 2024-11-16T14:57:07,105 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=11, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=dcd6414ddebad66e87ab37a6dba924f3, ASSIGN 2024-11-16T14:57:07,106 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=11, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=dcd6414ddebad66e87ab37a6dba924f3, ASSIGN; state=OFFLINE, location=fc3fda29f660,42985,1731769023951; forceNewPlan=false, retain=false 2024-11-16T14:57:07,107 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=10, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=a5db9431e22152bd2a82e84504d2879a, ASSIGN 2024-11-16T14:57:07,109 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=10, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=a5db9431e22152bd2a82e84504d2879a, ASSIGN; state=OFFLINE, location=fc3fda29f660,42985,1731769023951; forceNewPlan=false, retain=false 2024-11-16T14:57:07,110 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=9, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=fd3944fc2b78246a853fb2232c747c35, ASSIGN 2024-11-16T14:57:07,111 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=9, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=fd3944fc2b78246a853fb2232c747c35, ASSIGN; state=OFFLINE, location=fc3fda29f660,38897,1731769023879; forceNewPlan=false, retain=false 2024-11-16T14:57:07,113 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=8, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=dde7f54607f11c5fb4271e361e82dabd, ASSIGN 2024-11-16T14:57:07,114 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=8, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=dde7f54607f11c5fb4271e361e82dabd, ASSIGN; state=OFFLINE, location=fc3fda29f660,39089,1731769023769; forceNewPlan=false, retain=false 2024-11-16T14:57:07,115 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=7, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=5bb22144956dcd43c9b1d957232b11e7, ASSIGN 2024-11-16T14:57:07,117 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=7, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=5bb22144956dcd43c9b1d957232b11e7, ASSIGN; state=OFFLINE, location=fc3fda29f660,39089,1731769023769; forceNewPlan=false, retain=false 2024-11-16T14:57:07,118 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=6, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=c3dde5f5f668d914608a2987d4f4c001, ASSIGN 2024-11-16T14:57:07,119 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=6, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=c3dde5f5f668d914608a2987d4f4c001, ASSIGN; state=OFFLINE, location=fc3fda29f660,42985,1731769023951; forceNewPlan=false, retain=false 2024-11-16T14:57:07,121 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=4f0d635e77a1e3dd04c4d107bb0f1564, ASSIGN 2024-11-16T14:57:07,122 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=4f0d635e77a1e3dd04c4d107bb0f1564, ASSIGN; state=OFFLINE, location=fc3fda29f660,38897,1731769023879; forceNewPlan=false, retain=false 2024-11-16T14:57:07,208 INFO [fc3fda29f660:43615 {}] balancer.BaseLoadBalancer(388): Reassigned 26 regions. 26 retained the pre-restart assignment. 2024-11-16T14:57:07,210 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=28 updating hbase:meta row=dc4fcaece350cd66f6bf88cfbdff6f15, regionState=OPENING, regionLocation=fc3fda29f660,42985,1731769023951 2024-11-16T14:57:07,212 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=25 updating hbase:meta row=35ef9a7f35bb0b08ec1d53b82d3f9777, regionState=OPENING, regionLocation=fc3fda29f660,42985,1731769023951 2024-11-16T14:57:07,213 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=22 updating hbase:meta row=b364a006589fe0d2474ffa45692a6efc, regionState=OPENING, regionLocation=fc3fda29f660,42985,1731769023951 2024-11-16T14:57:07,214 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=19 updating hbase:meta row=299131947f15f9c44bcfe22cd6623096, regionState=OPENING, regionLocation=fc3fda29f660,42985,1731769023951 2024-11-16T14:57:07,214 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=16 updating hbase:meta row=266565e60d7419a2a30ceec1b0f6519b, regionState=OPENING, regionLocation=fc3fda29f660,42985,1731769023951 2024-11-16T14:57:07,215 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=12 updating hbase:meta row=1d97df67e5110b0fe1a1e4b7c1341cbb, regionState=OPENING, regionLocation=fc3fda29f660,42985,1731769023951 2024-11-16T14:57:07,215 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=11 updating hbase:meta row=dcd6414ddebad66e87ab37a6dba924f3, regionState=OPENING, regionLocation=fc3fda29f660,42985,1731769023951 2024-11-16T14:57:07,216 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=28, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=dc4fcaece350cd66f6bf88cfbdff6f15, ASSIGN because future has completed 2024-11-16T14:57:07,216 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=10 updating hbase:meta row=a5db9431e22152bd2a82e84504d2879a, regionState=OPENING, regionLocation=fc3fda29f660,42985,1731769023951 2024-11-16T14:57:07,217 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=31, ppid=28, state=RUNNABLE, hasLock=false; OpenRegionProcedure dc4fcaece350cd66f6bf88cfbdff6f15, server=fc3fda29f660,42985,1731769023951}] 2024-11-16T14:57:07,218 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=25, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=35ef9a7f35bb0b08ec1d53b82d3f9777, ASSIGN because future has completed 2024-11-16T14:57:07,220 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=22, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=b364a006589fe0d2474ffa45692a6efc, ASSIGN because future has completed 2024-11-16T14:57:07,222 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=19, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=299131947f15f9c44bcfe22cd6623096, ASSIGN because future has completed 2024-11-16T14:57:07,223 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=32, ppid=19, state=RUNNABLE, hasLock=false; OpenRegionProcedure 299131947f15f9c44bcfe22cd6623096, server=fc3fda29f660,42985,1731769023951}] 2024-11-16T14:57:07,224 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=16, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=266565e60d7419a2a30ceec1b0f6519b, ASSIGN because future has completed 2024-11-16T14:57:07,226 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=12, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=1d97df67e5110b0fe1a1e4b7c1341cbb, ASSIGN because future has completed 2024-11-16T14:57:07,228 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=11, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=dcd6414ddebad66e87ab37a6dba924f3, ASSIGN because future has completed 2024-11-16T14:57:07,230 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=10, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=a5db9431e22152bd2a82e84504d2879a, ASSIGN because future has completed 2024-11-16T14:57:07,230 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=33, ppid=10, state=RUNNABLE, hasLock=false; OpenRegionProcedure a5db9431e22152bd2a82e84504d2879a, server=fc3fda29f660,42985,1731769023951}] 2024-11-16T14:57:07,235 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=34, ppid=11, state=RUNNABLE, hasLock=false; OpenRegionProcedure dcd6414ddebad66e87ab37a6dba924f3, server=fc3fda29f660,42985,1731769023951}] 2024-11-16T14:57:07,242 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=35, ppid=12, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1d97df67e5110b0fe1a1e4b7c1341cbb, server=fc3fda29f660,42985,1731769023951}] 2024-11-16T14:57:07,247 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=36, ppid=16, state=RUNNABLE, hasLock=false; OpenRegionProcedure 266565e60d7419a2a30ceec1b0f6519b, server=fc3fda29f660,42985,1731769023951}] 2024-11-16T14:57:07,251 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=37, ppid=22, state=RUNNABLE, hasLock=false; OpenRegionProcedure b364a006589fe0d2474ffa45692a6efc, server=fc3fda29f660,42985,1731769023951}] 2024-11-16T14:57:07,256 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=38, ppid=25, state=RUNNABLE, hasLock=false; OpenRegionProcedure 35ef9a7f35bb0b08ec1d53b82d3f9777, server=fc3fda29f660,42985,1731769023951}] 2024-11-16T14:57:07,261 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=6 updating hbase:meta row=c3dde5f5f668d914608a2987d4f4c001, regionState=OPENING, regionLocation=fc3fda29f660,42985,1731769023951 2024-11-16T14:57:07,262 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=30 updating hbase:meta row=2f41a8c996224c818e917389bd3f5141, regionState=OPENING, regionLocation=fc3fda29f660,39089,1731769023769 2024-11-16T14:57:07,263 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=27 updating hbase:meta row=62a6d124ad9d161782c0632102c766de, regionState=OPENING, regionLocation=fc3fda29f660,39089,1731769023769 2024-11-16T14:57:07,263 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=24 updating hbase:meta row=7f11f79823720d664105d8643dfa862d, regionState=OPENING, regionLocation=fc3fda29f660,39089,1731769023769 2024-11-16T14:57:07,264 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=21 updating hbase:meta row=18f03e2509a3a89e4a69ba87fad73940, regionState=OPENING, regionLocation=fc3fda29f660,39089,1731769023769 2024-11-16T14:57:07,265 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=18 updating hbase:meta row=994ca1526cf6c4838e65f85444853f1a, regionState=OPENING, regionLocation=fc3fda29f660,39089,1731769023769 2024-11-16T14:57:07,265 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=c3dde5f5f668d914608a2987d4f4c001, ASSIGN because future has completed 2024-11-16T14:57:07,265 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=39, ppid=6, state=RUNNABLE, hasLock=false; OpenRegionProcedure c3dde5f5f668d914608a2987d4f4c001, server=fc3fda29f660,42985,1731769023951}] 2024-11-16T14:57:07,267 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=30, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=2f41a8c996224c818e917389bd3f5141, ASSIGN because future has completed 2024-11-16T14:57:07,269 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=27, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=62a6d124ad9d161782c0632102c766de, ASSIGN because future has completed 2024-11-16T14:57:07,271 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=40, ppid=27, state=RUNNABLE, hasLock=false; OpenRegionProcedure 62a6d124ad9d161782c0632102c766de, server=fc3fda29f660,39089,1731769023769}] 2024-11-16T14:57:07,271 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=24, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=7f11f79823720d664105d8643dfa862d, ASSIGN because future has completed 2024-11-16T14:57:07,273 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=21, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=18f03e2509a3a89e4a69ba87fad73940, ASSIGN because future has completed 2024-11-16T14:57:07,275 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=18, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=994ca1526cf6c4838e65f85444853f1a, ASSIGN because future has completed 2024-11-16T14:57:07,277 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=41, ppid=18, state=RUNNABLE, hasLock=false; OpenRegionProcedure 994ca1526cf6c4838e65f85444853f1a, server=fc3fda29f660,39089,1731769023769}] 2024-11-16T14:57:07,283 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=42, ppid=21, state=RUNNABLE, hasLock=false; OpenRegionProcedure 18f03e2509a3a89e4a69ba87fad73940, server=fc3fda29f660,39089,1731769023769}] 2024-11-16T14:57:07,288 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=43, ppid=24, state=RUNNABLE, hasLock=false; OpenRegionProcedure 7f11f79823720d664105d8643dfa862d, server=fc3fda29f660,39089,1731769023769}] 2024-11-16T14:57:07,293 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=44, ppid=30, state=RUNNABLE, hasLock=false; OpenRegionProcedure 2f41a8c996224c818e917389bd3f5141, server=fc3fda29f660,39089,1731769023769}] 2024-11-16T14:57:07,297 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=15 updating hbase:meta row=b4f73326a2a6a279d503e557c6ed452f, regionState=OPENING, regionLocation=fc3fda29f660,39089,1731769023769 2024-11-16T14:57:07,298 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=8 updating hbase:meta row=dde7f54607f11c5fb4271e361e82dabd, regionState=OPENING, regionLocation=fc3fda29f660,39089,1731769023769 2024-11-16T14:57:07,299 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=5bb22144956dcd43c9b1d957232b11e7, regionState=OPENING, regionLocation=fc3fda29f660,39089,1731769023769 2024-11-16T14:57:07,300 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=29 updating hbase:meta row=426ff231e487ce9f8492776ca1dacc85, regionState=OPENING, regionLocation=fc3fda29f660,38897,1731769023879 2024-11-16T14:57:07,300 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=15, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=b4f73326a2a6a279d503e557c6ed452f, ASSIGN because future has completed 2024-11-16T14:57:07,301 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=45, ppid=15, state=RUNNABLE, hasLock=false; OpenRegionProcedure b4f73326a2a6a279d503e557c6ed452f, server=fc3fda29f660,39089,1731769023769}] 2024-11-16T14:57:07,302 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=8, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=dde7f54607f11c5fb4271e361e82dabd, ASSIGN because future has completed 2024-11-16T14:57:07,304 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=7, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=5bb22144956dcd43c9b1d957232b11e7, ASSIGN because future has completed 2024-11-16T14:57:07,305 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=29, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=426ff231e487ce9f8492776ca1dacc85, ASSIGN because future has completed 2024-11-16T14:57:07,306 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=46, ppid=29, state=RUNNABLE, hasLock=false; OpenRegionProcedure 426ff231e487ce9f8492776ca1dacc85, server=fc3fda29f660,38897,1731769023879}] 2024-11-16T14:57:07,311 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=47, ppid=7, state=RUNNABLE, hasLock=false; OpenRegionProcedure 5bb22144956dcd43c9b1d957232b11e7, server=fc3fda29f660,39089,1731769023769}] 2024-11-16T14:57:07,317 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=48, ppid=8, state=RUNNABLE, hasLock=false; OpenRegionProcedure dde7f54607f11c5fb4271e361e82dabd, server=fc3fda29f660,39089,1731769023769}] 2024-11-16T14:57:07,321 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=26 updating hbase:meta row=b5566c7e10a0e001c7f58316bdd43c2f, regionState=OPENING, regionLocation=fc3fda29f660,38897,1731769023879 2024-11-16T14:57:07,322 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=23 updating hbase:meta row=33054df03c1be51dc860405880be6075, regionState=OPENING, regionLocation=fc3fda29f660,38897,1731769023879 2024-11-16T14:57:07,322 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=20 updating hbase:meta row=7f5c9df3a544933227f58b052f743f72, regionState=OPENING, regionLocation=fc3fda29f660,38897,1731769023879 2024-11-16T14:57:07,323 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=17 updating hbase:meta row=c2acbc05ae7bb1607a5719b701b40383, regionState=OPENING, regionLocation=fc3fda29f660,38897,1731769023879 2024-11-16T14:57:07,323 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=14 updating hbase:meta row=7eb7b75d28b808c5c3f2f96e407b4347, regionState=OPENING, regionLocation=fc3fda29f660,38897,1731769023879 2024-11-16T14:57:07,324 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=13 updating hbase:meta row=7352dcdbf9811dcebc51a714a1de4f55, regionState=OPENING, regionLocation=fc3fda29f660,38897,1731769023879 2024-11-16T14:57:07,324 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=9 updating hbase:meta row=fd3944fc2b78246a853fb2232c747c35, regionState=OPENING, regionLocation=fc3fda29f660,38897,1731769023879 2024-11-16T14:57:07,324 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=26, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=b5566c7e10a0e001c7f58316bdd43c2f, ASSIGN because future has completed 2024-11-16T14:57:07,325 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=49, ppid=26, state=RUNNABLE, hasLock=false; OpenRegionProcedure b5566c7e10a0e001c7f58316bdd43c2f, server=fc3fda29f660,38897,1731769023879}] 2024-11-16T14:57:07,326 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=23, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=33054df03c1be51dc860405880be6075, ASSIGN because future has completed 2024-11-16T14:57:07,327 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=20, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=7f5c9df3a544933227f58b052f743f72, ASSIGN because future has completed 2024-11-16T14:57:07,328 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=17, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=c2acbc05ae7bb1607a5719b701b40383, ASSIGN because future has completed 2024-11-16T14:57:07,329 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=14, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=7eb7b75d28b808c5c3f2f96e407b4347, ASSIGN because future has completed 2024-11-16T14:57:07,330 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=50, ppid=14, state=RUNNABLE, hasLock=false; OpenRegionProcedure 7eb7b75d28b808c5c3f2f96e407b4347, server=fc3fda29f660,38897,1731769023879}] 2024-11-16T14:57:07,331 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=13, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=7352dcdbf9811dcebc51a714a1de4f55, ASSIGN because future has completed 2024-11-16T14:57:07,332 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=9, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=fd3944fc2b78246a853fb2232c747c35, ASSIGN because future has completed 2024-11-16T14:57:07,335 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=51, ppid=9, state=RUNNABLE, hasLock=false; OpenRegionProcedure fd3944fc2b78246a853fb2232c747c35, server=fc3fda29f660,38897,1731769023879}] 2024-11-16T14:57:07,339 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=52, ppid=13, state=RUNNABLE, hasLock=false; OpenRegionProcedure 7352dcdbf9811dcebc51a714a1de4f55, server=fc3fda29f660,38897,1731769023879}] 2024-11-16T14:57:07,343 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=53, ppid=17, state=RUNNABLE, hasLock=false; OpenRegionProcedure c2acbc05ae7bb1607a5719b701b40383, server=fc3fda29f660,38897,1731769023879}] 2024-11-16T14:57:07,348 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=54, ppid=20, state=RUNNABLE, hasLock=false; OpenRegionProcedure 7f5c9df3a544933227f58b052f743f72, server=fc3fda29f660,38897,1731769023879}] 2024-11-16T14:57:07,352 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=55, ppid=23, state=RUNNABLE, hasLock=false; OpenRegionProcedure 33054df03c1be51dc860405880be6075, server=fc3fda29f660,38897,1731769023879}] 2024-11-16T14:57:07,356 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=4f0d635e77a1e3dd04c4d107bb0f1564, regionState=OPENING, regionLocation=fc3fda29f660,38897,1731769023879 2024-11-16T14:57:07,359 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=4f0d635e77a1e3dd04c4d107bb0f1564, ASSIGN because future has completed 2024-11-16T14:57:07,359 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=56, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 4f0d635e77a1e3dd04c4d107bb0f1564, server=fc3fda29f660,38897,1731769023879}] 2024-11-16T14:57:07,372 DEBUG [RSProcedureDispatcher-pool-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-16T14:57:07,374 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46991, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-16T14:57:07,381 INFO [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] handler.AssignRegionHandler(132): Open testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,hhh,1731769026392.1d97df67e5110b0fe1a1e4b7c1341cbb. 2024-11-16T14:57:07,381 DEBUG [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(7752): Opening region: {ENCODED => 1d97df67e5110b0fe1a1e4b7c1341cbb, NAME => 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,hhh,1731769026392.1d97df67e5110b0fe1a1e4b7c1341cbb.', STARTKEY => 'hhh', ENDKEY => 'iii'} 2024-11-16T14:57:07,382 DEBUG [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true 1d97df67e5110b0fe1a1e4b7c1341cbb 2024-11-16T14:57:07,382 DEBUG [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(898): Instantiated testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,hhh,1731769026392.1d97df67e5110b0fe1a1e4b7c1341cbb.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-16T14:57:07,382 DEBUG [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(7794): checking encryption for 1d97df67e5110b0fe1a1e4b7c1341cbb 2024-11-16T14:57:07,382 DEBUG [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(7797): checking classloading for 1d97df67e5110b0fe1a1e4b7c1341cbb 2024-11-16T14:57:07,385 INFO [StoreOpener-1d97df67e5110b0fe1a1e4b7c1341cbb-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily11 of region 1d97df67e5110b0fe1a1e4b7c1341cbb 2024-11-16T14:57:07,387 INFO [StoreOpener-1d97df67e5110b0fe1a1e4b7c1341cbb-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1d97df67e5110b0fe1a1e4b7c1341cbb columnFamilyName colfamily11 2024-11-16T14:57:07,387 DEBUG [StoreOpener-1d97df67e5110b0fe1a1e4b7c1341cbb-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:57:07,388 INFO [StoreOpener-1d97df67e5110b0fe1a1e4b7c1341cbb-1 {}] regionserver.HStore(327): Store=1d97df67e5110b0fe1a1e4b7c1341cbb/colfamily11, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-16T14:57:07,388 INFO [StoreOpener-1d97df67e5110b0fe1a1e4b7c1341cbb-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily21 of region 1d97df67e5110b0fe1a1e4b7c1341cbb 2024-11-16T14:57:07,390 INFO [StoreOpener-1d97df67e5110b0fe1a1e4b7c1341cbb-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1d97df67e5110b0fe1a1e4b7c1341cbb columnFamilyName colfamily21 2024-11-16T14:57:07,390 DEBUG [StoreOpener-1d97df67e5110b0fe1a1e4b7c1341cbb-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:57:07,391 INFO [StoreOpener-1d97df67e5110b0fe1a1e4b7c1341cbb-1 {}] regionserver.HStore(327): Store=1d97df67e5110b0fe1a1e4b7c1341cbb/colfamily21, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-16T14:57:07,391 INFO [StoreOpener-1d97df67e5110b0fe1a1e4b7c1341cbb-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily31 of region 1d97df67e5110b0fe1a1e4b7c1341cbb 2024-11-16T14:57:07,393 INFO [StoreOpener-1d97df67e5110b0fe1a1e4b7c1341cbb-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1d97df67e5110b0fe1a1e4b7c1341cbb columnFamilyName colfamily31 2024-11-16T14:57:07,393 DEBUG [StoreOpener-1d97df67e5110b0fe1a1e4b7c1341cbb-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:57:07,394 INFO [StoreOpener-1d97df67e5110b0fe1a1e4b7c1341cbb-1 {}] regionserver.HStore(327): Store=1d97df67e5110b0fe1a1e4b7c1341cbb/colfamily31, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-16T14:57:07,394 DEBUG [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(1038): replaying wal for 1d97df67e5110b0fe1a1e4b7c1341cbb 2024-11-16T14:57:07,395 DEBUG [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42223/user/jenkins/test-data/a8176cb1-2ebb-b530-b3c7-53c9e53c186e/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/1d97df67e5110b0fe1a1e4b7c1341cbb 2024-11-16T14:57:07,396 DEBUG [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42223/user/jenkins/test-data/a8176cb1-2ebb-b530-b3c7-53c9e53c186e/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/1d97df67e5110b0fe1a1e4b7c1341cbb 2024-11-16T14:57:07,397 DEBUG [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(1048): stopping wal replay for 1d97df67e5110b0fe1a1e4b7c1341cbb 2024-11-16T14:57:07,397 DEBUG [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(1060): Cleaning up temporary data for 1d97df67e5110b0fe1a1e4b7c1341cbb 2024-11-16T14:57:07,398 DEBUG [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-16T14:57:07,400 DEBUG [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(1093): writing seq id for 1d97df67e5110b0fe1a1e4b7c1341cbb 2024-11-16T14:57:07,403 DEBUG [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:42223/user/jenkins/test-data/a8176cb1-2ebb-b530-b3c7-53c9e53c186e/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/1d97df67e5110b0fe1a1e4b7c1341cbb/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-16T14:57:07,404 INFO [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(1114): Opened 1d97df67e5110b0fe1a1e4b7c1341cbb; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=72303480, jitterRate=0.07740581035614014}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-16T14:57:07,404 DEBUG [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1d97df67e5110b0fe1a1e4b7c1341cbb 2024-11-16T14:57:07,405 DEBUG [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(1006): Region open journal for 1d97df67e5110b0fe1a1e4b7c1341cbb: Running coprocessor pre-open hook at 1731769027382Writing region info on filesystem at 1731769027382Initializing all the Stores at 1731769027384 (+2 ms)Instantiating store for column family {NAME => 'colfamily11', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731769027384Instantiating store for column family {NAME => 'colfamily21', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731769027385 (+1 ms)Instantiating store for column family {NAME => 'colfamily31', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731769027385Cleaning up temporary data from old regions at 1731769027397 (+12 ms)Running coprocessor post-open hooks at 1731769027404 (+7 ms)Region opened successfully at 1731769027405 (+1 ms) 2024-11-16T14:57:07,407 INFO [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegionServer(2236): Post open deploy tasks for testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,hhh,1731769026392.1d97df67e5110b0fe1a1e4b7c1341cbb., pid=35, masterSystemTime=1731769027371 2024-11-16T14:57:07,410 DEBUG [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegionServer(2266): Finished post open deploy task for testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,hhh,1731769026392.1d97df67e5110b0fe1a1e4b7c1341cbb. 2024-11-16T14:57:07,410 INFO [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] handler.AssignRegionHandler(153): Opened testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,hhh,1731769026392.1d97df67e5110b0fe1a1e4b7c1341cbb. 2024-11-16T14:57:07,410 INFO [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] handler.AssignRegionHandler(132): Open testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,rrr,1731769026392.b364a006589fe0d2474ffa45692a6efc. 2024-11-16T14:57:07,411 DEBUG [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(7752): Opening region: {ENCODED => b364a006589fe0d2474ffa45692a6efc, NAME => 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,rrr,1731769026392.b364a006589fe0d2474ffa45692a6efc.', STARTKEY => 'rrr', ENDKEY => 'sss'} 2024-11-16T14:57:07,411 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=12 updating hbase:meta row=1d97df67e5110b0fe1a1e4b7c1341cbb, regionState=OPEN, openSeqNum=2, regionLocation=fc3fda29f660,42985,1731769023951 2024-11-16T14:57:07,411 DEBUG [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true b364a006589fe0d2474ffa45692a6efc 2024-11-16T14:57:07,411 DEBUG [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(898): Instantiated testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,rrr,1731769026392.b364a006589fe0d2474ffa45692a6efc.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-16T14:57:07,412 DEBUG [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(7794): checking encryption for b364a006589fe0d2474ffa45692a6efc 2024-11-16T14:57:07,412 DEBUG [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(7797): checking classloading for b364a006589fe0d2474ffa45692a6efc 2024-11-16T14:57:07,414 INFO [StoreOpener-b364a006589fe0d2474ffa45692a6efc-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily11 of region b364a006589fe0d2474ffa45692a6efc 2024-11-16T14:57:07,414 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=35, ppid=12, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1d97df67e5110b0fe1a1e4b7c1341cbb, server=fc3fda29f660,42985,1731769023951 because future has completed 2024-11-16T14:57:07,417 INFO [StoreOpener-b364a006589fe0d2474ffa45692a6efc-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region b364a006589fe0d2474ffa45692a6efc columnFamilyName colfamily11 2024-11-16T14:57:07,417 DEBUG [StoreOpener-b364a006589fe0d2474ffa45692a6efc-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:57:07,418 INFO [StoreOpener-b364a006589fe0d2474ffa45692a6efc-1 {}] regionserver.HStore(327): Store=b364a006589fe0d2474ffa45692a6efc/colfamily11, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-16T14:57:07,418 INFO [StoreOpener-b364a006589fe0d2474ffa45692a6efc-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily21 of region b364a006589fe0d2474ffa45692a6efc 2024-11-16T14:57:07,420 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=35, resume processing ppid=12 2024-11-16T14:57:07,420 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=35, ppid=12, state=SUCCESS, hasLock=false; OpenRegionProcedure 1d97df67e5110b0fe1a1e4b7c1341cbb, server=fc3fda29f660,42985,1731769023951 in 174 msec 2024-11-16T14:57:07,420 INFO [StoreOpener-b364a006589fe0d2474ffa45692a6efc-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region b364a006589fe0d2474ffa45692a6efc columnFamilyName colfamily21 2024-11-16T14:57:07,420 DEBUG [StoreOpener-b364a006589fe0d2474ffa45692a6efc-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:57:07,421 INFO [StoreOpener-b364a006589fe0d2474ffa45692a6efc-1 {}] regionserver.HStore(327): Store=b364a006589fe0d2474ffa45692a6efc/colfamily21, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-16T14:57:07,421 INFO [StoreOpener-b364a006589fe0d2474ffa45692a6efc-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily31 of region b364a006589fe0d2474ffa45692a6efc 2024-11-16T14:57:07,423 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=12, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=1d97df67e5110b0fe1a1e4b7c1341cbb, ASSIGN in 382 msec 2024-11-16T14:57:07,424 INFO [StoreOpener-b364a006589fe0d2474ffa45692a6efc-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region b364a006589fe0d2474ffa45692a6efc columnFamilyName colfamily31 2024-11-16T14:57:07,424 DEBUG [StoreOpener-b364a006589fe0d2474ffa45692a6efc-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:57:07,425 INFO [StoreOpener-b364a006589fe0d2474ffa45692a6efc-1 {}] regionserver.HStore(327): Store=b364a006589fe0d2474ffa45692a6efc/colfamily31, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-16T14:57:07,425 DEBUG [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1038): replaying wal for b364a006589fe0d2474ffa45692a6efc 2024-11-16T14:57:07,425 DEBUG [RSProcedureDispatcher-pool-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-16T14:57:07,426 DEBUG [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42223/user/jenkins/test-data/a8176cb1-2ebb-b530-b3c7-53c9e53c186e/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/b364a006589fe0d2474ffa45692a6efc 2024-11-16T14:57:07,427 DEBUG [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42223/user/jenkins/test-data/a8176cb1-2ebb-b530-b3c7-53c9e53c186e/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/b364a006589fe0d2474ffa45692a6efc 2024-11-16T14:57:07,427 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34169, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-16T14:57:07,429 DEBUG [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1048): stopping wal replay for b364a006589fe0d2474ffa45692a6efc 2024-11-16T14:57:07,429 DEBUG [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1060): Cleaning up temporary data for b364a006589fe0d2474ffa45692a6efc 2024-11-16T14:57:07,429 DEBUG [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-16T14:57:07,431 DEBUG [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1093): writing seq id for b364a006589fe0d2474ffa45692a6efc 2024-11-16T14:57:07,432 INFO [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] handler.AssignRegionHandler(132): Open testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,qqq,1731769026392.18f03e2509a3a89e4a69ba87fad73940. 2024-11-16T14:57:07,433 DEBUG [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(7752): Opening region: {ENCODED => 18f03e2509a3a89e4a69ba87fad73940, NAME => 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,qqq,1731769026392.18f03e2509a3a89e4a69ba87fad73940.', STARTKEY => 'qqq', ENDKEY => 'rrr'} 2024-11-16T14:57:07,433 DEBUG [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true 18f03e2509a3a89e4a69ba87fad73940 2024-11-16T14:57:07,433 DEBUG [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(898): Instantiated testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,qqq,1731769026392.18f03e2509a3a89e4a69ba87fad73940.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-16T14:57:07,433 DEBUG [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(7794): checking encryption for 18f03e2509a3a89e4a69ba87fad73940 2024-11-16T14:57:07,434 DEBUG [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(7797): checking classloading for 18f03e2509a3a89e4a69ba87fad73940 2024-11-16T14:57:07,435 DEBUG [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:42223/user/jenkins/test-data/a8176cb1-2ebb-b530-b3c7-53c9e53c186e/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/b364a006589fe0d2474ffa45692a6efc/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-16T14:57:07,435 INFO [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1114): Opened b364a006589fe0d2474ffa45692a6efc; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=72455075, jitterRate=0.07966475188732147}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-16T14:57:07,436 DEBUG [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1122): Running coprocessor post-open hooks for b364a006589fe0d2474ffa45692a6efc 2024-11-16T14:57:07,436 DEBUG [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1006): Region open journal for b364a006589fe0d2474ffa45692a6efc: Running coprocessor pre-open hook at 1731769027412Writing region info on filesystem at 1731769027412Initializing all the Stores at 1731769027414 (+2 ms)Instantiating store for column family {NAME => 'colfamily11', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731769027414Instantiating store for column family {NAME => 'colfamily21', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731769027414Instantiating store for column family {NAME => 'colfamily31', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731769027414Cleaning up temporary data from old regions at 1731769027429 (+15 ms)Running coprocessor post-open hooks at 1731769027436 (+7 ms)Region opened successfully at 1731769027436 2024-11-16T14:57:07,436 INFO [StoreOpener-18f03e2509a3a89e4a69ba87fad73940-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily11 of region 18f03e2509a3a89e4a69ba87fad73940 2024-11-16T14:57:07,437 INFO [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegionServer(2236): Post open deploy tasks for testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,rrr,1731769026392.b364a006589fe0d2474ffa45692a6efc., pid=37, masterSystemTime=1731769027371 2024-11-16T14:57:07,438 INFO [StoreOpener-18f03e2509a3a89e4a69ba87fad73940-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 18f03e2509a3a89e4a69ba87fad73940 columnFamilyName colfamily11 2024-11-16T14:57:07,438 DEBUG [StoreOpener-18f03e2509a3a89e4a69ba87fad73940-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:57:07,439 INFO [StoreOpener-18f03e2509a3a89e4a69ba87fad73940-1 {}] regionserver.HStore(327): Store=18f03e2509a3a89e4a69ba87fad73940/colfamily11, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-16T14:57:07,439 INFO [StoreOpener-18f03e2509a3a89e4a69ba87fad73940-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily21 of region 18f03e2509a3a89e4a69ba87fad73940 2024-11-16T14:57:07,440 DEBUG [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegionServer(2266): Finished post open deploy task for testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,rrr,1731769026392.b364a006589fe0d2474ffa45692a6efc. 2024-11-16T14:57:07,440 INFO [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] handler.AssignRegionHandler(153): Opened testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,rrr,1731769026392.b364a006589fe0d2474ffa45692a6efc. 2024-11-16T14:57:07,441 INFO [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] handler.AssignRegionHandler(132): Open testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ggg,1731769026392.dcd6414ddebad66e87ab37a6dba924f3. 2024-11-16T14:57:07,441 DEBUG [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(7752): Opening region: {ENCODED => dcd6414ddebad66e87ab37a6dba924f3, NAME => 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ggg,1731769026392.dcd6414ddebad66e87ab37a6dba924f3.', STARTKEY => 'ggg', ENDKEY => 'hhh'} 2024-11-16T14:57:07,442 INFO [StoreOpener-18f03e2509a3a89e4a69ba87fad73940-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 18f03e2509a3a89e4a69ba87fad73940 columnFamilyName colfamily21 2024-11-16T14:57:07,442 DEBUG [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true dcd6414ddebad66e87ab37a6dba924f3 2024-11-16T14:57:07,442 DEBUG [StoreOpener-18f03e2509a3a89e4a69ba87fad73940-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:57:07,442 DEBUG [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(898): Instantiated testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ggg,1731769026392.dcd6414ddebad66e87ab37a6dba924f3.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-16T14:57:07,442 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=22 updating hbase:meta row=b364a006589fe0d2474ffa45692a6efc, regionState=OPEN, openSeqNum=2, regionLocation=fc3fda29f660,42985,1731769023951 2024-11-16T14:57:07,442 DEBUG [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(7794): checking encryption for dcd6414ddebad66e87ab37a6dba924f3 2024-11-16T14:57:07,442 DEBUG [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(7797): checking classloading for dcd6414ddebad66e87ab37a6dba924f3 2024-11-16T14:57:07,443 INFO [StoreOpener-18f03e2509a3a89e4a69ba87fad73940-1 {}] regionserver.HStore(327): Store=18f03e2509a3a89e4a69ba87fad73940/colfamily21, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-16T14:57:07,443 INFO [StoreOpener-18f03e2509a3a89e4a69ba87fad73940-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily31 of region 18f03e2509a3a89e4a69ba87fad73940 2024-11-16T14:57:07,445 INFO [StoreOpener-dcd6414ddebad66e87ab37a6dba924f3-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily11 of region dcd6414ddebad66e87ab37a6dba924f3 2024-11-16T14:57:07,445 INFO [StoreOpener-18f03e2509a3a89e4a69ba87fad73940-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 18f03e2509a3a89e4a69ba87fad73940 columnFamilyName colfamily31 2024-11-16T14:57:07,446 DEBUG [StoreOpener-18f03e2509a3a89e4a69ba87fad73940-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:57:07,446 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=37, ppid=22, state=RUNNABLE, hasLock=false; OpenRegionProcedure b364a006589fe0d2474ffa45692a6efc, server=fc3fda29f660,42985,1731769023951 because future has completed 2024-11-16T14:57:07,446 INFO [StoreOpener-18f03e2509a3a89e4a69ba87fad73940-1 {}] regionserver.HStore(327): Store=18f03e2509a3a89e4a69ba87fad73940/colfamily31, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-16T14:57:07,447 DEBUG [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(1038): replaying wal for 18f03e2509a3a89e4a69ba87fad73940 2024-11-16T14:57:07,448 INFO [StoreOpener-dcd6414ddebad66e87ab37a6dba924f3-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region dcd6414ddebad66e87ab37a6dba924f3 columnFamilyName colfamily11 2024-11-16T14:57:07,448 DEBUG [StoreOpener-dcd6414ddebad66e87ab37a6dba924f3-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:57:07,448 DEBUG [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42223/user/jenkins/test-data/a8176cb1-2ebb-b530-b3c7-53c9e53c186e/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/18f03e2509a3a89e4a69ba87fad73940 2024-11-16T14:57:07,448 INFO [StoreOpener-dcd6414ddebad66e87ab37a6dba924f3-1 {}] regionserver.HStore(327): Store=dcd6414ddebad66e87ab37a6dba924f3/colfamily11, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-16T14:57:07,449 INFO [StoreOpener-dcd6414ddebad66e87ab37a6dba924f3-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily21 of region dcd6414ddebad66e87ab37a6dba924f3 2024-11-16T14:57:07,449 DEBUG [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42223/user/jenkins/test-data/a8176cb1-2ebb-b530-b3c7-53c9e53c186e/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/18f03e2509a3a89e4a69ba87fad73940 2024-11-16T14:57:07,451 INFO [StoreOpener-dcd6414ddebad66e87ab37a6dba924f3-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region dcd6414ddebad66e87ab37a6dba924f3 columnFamilyName colfamily21 2024-11-16T14:57:07,451 DEBUG [StoreOpener-dcd6414ddebad66e87ab37a6dba924f3-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:57:07,451 DEBUG [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(1048): stopping wal replay for 18f03e2509a3a89e4a69ba87fad73940 2024-11-16T14:57:07,451 DEBUG [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(1060): Cleaning up temporary data for 18f03e2509a3a89e4a69ba87fad73940 2024-11-16T14:57:07,452 INFO [StoreOpener-dcd6414ddebad66e87ab37a6dba924f3-1 {}] regionserver.HStore(327): Store=dcd6414ddebad66e87ab37a6dba924f3/colfamily21, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-16T14:57:07,452 DEBUG [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-16T14:57:07,452 INFO [StoreOpener-dcd6414ddebad66e87ab37a6dba924f3-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily31 of region dcd6414ddebad66e87ab37a6dba924f3 2024-11-16T14:57:07,452 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=37, resume processing ppid=22 2024-11-16T14:57:07,452 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=37, ppid=22, state=SUCCESS, hasLock=false; OpenRegionProcedure b364a006589fe0d2474ffa45692a6efc, server=fc3fda29f660,42985,1731769023951 in 196 msec 2024-11-16T14:57:07,454 INFO [StoreOpener-dcd6414ddebad66e87ab37a6dba924f3-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region dcd6414ddebad66e87ab37a6dba924f3 columnFamilyName colfamily31 2024-11-16T14:57:07,454 DEBUG [StoreOpener-dcd6414ddebad66e87ab37a6dba924f3-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:57:07,454 DEBUG [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(1093): writing seq id for 18f03e2509a3a89e4a69ba87fad73940 2024-11-16T14:57:07,455 INFO [StoreOpener-dcd6414ddebad66e87ab37a6dba924f3-1 {}] regionserver.HStore(327): Store=dcd6414ddebad66e87ab37a6dba924f3/colfamily31, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-16T14:57:07,455 DEBUG [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1038): replaying wal for dcd6414ddebad66e87ab37a6dba924f3 2024-11-16T14:57:07,457 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=22, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=b364a006589fe0d2474ffa45692a6efc, ASSIGN in 414 msec 2024-11-16T14:57:07,457 DEBUG [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42223/user/jenkins/test-data/a8176cb1-2ebb-b530-b3c7-53c9e53c186e/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/dcd6414ddebad66e87ab37a6dba924f3 2024-11-16T14:57:07,458 DEBUG [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42223/user/jenkins/test-data/a8176cb1-2ebb-b530-b3c7-53c9e53c186e/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/dcd6414ddebad66e87ab37a6dba924f3 2024-11-16T14:57:07,458 DEBUG [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:42223/user/jenkins/test-data/a8176cb1-2ebb-b530-b3c7-53c9e53c186e/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/18f03e2509a3a89e4a69ba87fad73940/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-16T14:57:07,459 INFO [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(1114): Opened 18f03e2509a3a89e4a69ba87fad73940; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=59104083, jitterRate=-0.11928053200244904}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-16T14:57:07,459 DEBUG [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 18f03e2509a3a89e4a69ba87fad73940 2024-11-16T14:57:07,459 DEBUG [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(1006): Region open journal for 18f03e2509a3a89e4a69ba87fad73940: Running coprocessor pre-open hook at 1731769027434Writing region info on filesystem at 1731769027434Initializing all the Stores at 1731769027435 (+1 ms)Instantiating store for column family {NAME => 'colfamily11', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731769027435Instantiating store for column family {NAME => 'colfamily21', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731769027436 (+1 ms)Instantiating store for column family {NAME => 'colfamily31', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731769027436Cleaning up temporary data from old regions at 1731769027451 (+15 ms)Running coprocessor post-open hooks at 1731769027459 (+8 ms)Region opened successfully at 1731769027459 2024-11-16T14:57:07,459 DEBUG [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1048): stopping wal replay for dcd6414ddebad66e87ab37a6dba924f3 2024-11-16T14:57:07,459 DEBUG [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1060): Cleaning up temporary data for dcd6414ddebad66e87ab37a6dba924f3 2024-11-16T14:57:07,460 DEBUG [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-16T14:57:07,460 INFO [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegionServer(2236): Post open deploy tasks for testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,qqq,1731769026392.18f03e2509a3a89e4a69ba87fad73940., pid=42, masterSystemTime=1731769027425 2024-11-16T14:57:07,462 DEBUG [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1093): writing seq id for dcd6414ddebad66e87ab37a6dba924f3 2024-11-16T14:57:07,464 DEBUG [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegionServer(2266): Finished post open deploy task for testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,qqq,1731769026392.18f03e2509a3a89e4a69ba87fad73940. 2024-11-16T14:57:07,464 INFO [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] handler.AssignRegionHandler(153): Opened testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,qqq,1731769026392.18f03e2509a3a89e4a69ba87fad73940. 2024-11-16T14:57:07,464 INFO [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] handler.AssignRegionHandler(132): Open testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ccc,1731769026392.5bb22144956dcd43c9b1d957232b11e7. 2024-11-16T14:57:07,465 DEBUG [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(7752): Opening region: {ENCODED => 5bb22144956dcd43c9b1d957232b11e7, NAME => 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ccc,1731769026392.5bb22144956dcd43c9b1d957232b11e7.', STARTKEY => 'ccc', ENDKEY => 'ddd'} 2024-11-16T14:57:07,465 DEBUG [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true 5bb22144956dcd43c9b1d957232b11e7 2024-11-16T14:57:07,465 DEBUG [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(898): Instantiated testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ccc,1731769026392.5bb22144956dcd43c9b1d957232b11e7.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-16T14:57:07,466 DEBUG [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(7794): checking encryption for 5bb22144956dcd43c9b1d957232b11e7 2024-11-16T14:57:07,466 DEBUG [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(7797): checking classloading for 5bb22144956dcd43c9b1d957232b11e7 2024-11-16T14:57:07,466 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=21 updating hbase:meta row=18f03e2509a3a89e4a69ba87fad73940, regionState=OPEN, openSeqNum=2, regionLocation=fc3fda29f660,39089,1731769023769 2024-11-16T14:57:07,466 INFO [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] handler.AssignRegionHandler(132): Open testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,mmm,1731769026392.c2acbc05ae7bb1607a5719b701b40383. 2024-11-16T14:57:07,467 DEBUG [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(7752): Opening region: {ENCODED => c2acbc05ae7bb1607a5719b701b40383, NAME => 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,mmm,1731769026392.c2acbc05ae7bb1607a5719b701b40383.', STARTKEY => 'mmm', ENDKEY => 'nnn'} 2024-11-16T14:57:07,467 DEBUG [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true c2acbc05ae7bb1607a5719b701b40383 2024-11-16T14:57:07,468 DEBUG [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(898): Instantiated testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,mmm,1731769026392.c2acbc05ae7bb1607a5719b701b40383.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-16T14:57:07,468 DEBUG [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(7794): checking encryption for c2acbc05ae7bb1607a5719b701b40383 2024-11-16T14:57:07,468 DEBUG [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(7797): checking classloading for c2acbc05ae7bb1607a5719b701b40383 2024-11-16T14:57:07,470 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=42, ppid=21, state=RUNNABLE, hasLock=false; OpenRegionProcedure 18f03e2509a3a89e4a69ba87fad73940, server=fc3fda29f660,39089,1731769023769 because future has completed 2024-11-16T14:57:07,474 DEBUG [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:42223/user/jenkins/test-data/a8176cb1-2ebb-b530-b3c7-53c9e53c186e/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/dcd6414ddebad66e87ab37a6dba924f3/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-16T14:57:07,475 INFO [StoreOpener-5bb22144956dcd43c9b1d957232b11e7-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily11 of region 5bb22144956dcd43c9b1d957232b11e7 2024-11-16T14:57:07,475 INFO [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1114): Opened dcd6414ddebad66e87ab37a6dba924f3; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=59383331, jitterRate=-0.11511941254138947}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-16T14:57:07,475 DEBUG [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1122): Running coprocessor post-open hooks for dcd6414ddebad66e87ab37a6dba924f3 2024-11-16T14:57:07,475 DEBUG [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1006): Region open journal for dcd6414ddebad66e87ab37a6dba924f3: Running coprocessor pre-open hook at 1731769027442Writing region info on filesystem at 1731769027442Initializing all the Stores at 1731769027444 (+2 ms)Instantiating store for column family {NAME => 'colfamily11', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731769027444Instantiating store for column family {NAME => 'colfamily21', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731769027445 (+1 ms)Instantiating store for column family {NAME => 'colfamily31', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731769027445Cleaning up temporary data from old regions at 1731769027459 (+14 ms)Running coprocessor post-open hooks at 1731769027475 (+16 ms)Region opened successfully at 1731769027475 2024-11-16T14:57:07,476 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=42, resume processing ppid=21 2024-11-16T14:57:07,476 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=42, ppid=21, state=SUCCESS, hasLock=false; OpenRegionProcedure 18f03e2509a3a89e4a69ba87fad73940, server=fc3fda29f660,39089,1731769023769 in 189 msec 2024-11-16T14:57:07,479 INFO [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegionServer(2236): Post open deploy tasks for testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ggg,1731769026392.dcd6414ddebad66e87ab37a6dba924f3., pid=34, masterSystemTime=1731769027371 2024-11-16T14:57:07,479 INFO [StoreOpener-c2acbc05ae7bb1607a5719b701b40383-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily11 of region c2acbc05ae7bb1607a5719b701b40383 2024-11-16T14:57:07,479 INFO [StoreOpener-5bb22144956dcd43c9b1d957232b11e7-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 5bb22144956dcd43c9b1d957232b11e7 columnFamilyName colfamily11 2024-11-16T14:57:07,479 DEBUG [StoreOpener-5bb22144956dcd43c9b1d957232b11e7-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:57:07,481 INFO [StoreOpener-5bb22144956dcd43c9b1d957232b11e7-1 {}] regionserver.HStore(327): Store=5bb22144956dcd43c9b1d957232b11e7/colfamily11, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-16T14:57:07,482 INFO [StoreOpener-5bb22144956dcd43c9b1d957232b11e7-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily21 of region 5bb22144956dcd43c9b1d957232b11e7 2024-11-16T14:57:07,482 INFO [StoreOpener-c2acbc05ae7bb1607a5719b701b40383-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region c2acbc05ae7bb1607a5719b701b40383 columnFamilyName colfamily11 2024-11-16T14:57:07,483 DEBUG [StoreOpener-c2acbc05ae7bb1607a5719b701b40383-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:57:07,483 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=21, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=18f03e2509a3a89e4a69ba87fad73940, ASSIGN in 437 msec 2024-11-16T14:57:07,484 INFO [StoreOpener-c2acbc05ae7bb1607a5719b701b40383-1 {}] regionserver.HStore(327): Store=c2acbc05ae7bb1607a5719b701b40383/colfamily11, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-16T14:57:07,484 INFO [StoreOpener-c2acbc05ae7bb1607a5719b701b40383-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily21 of region c2acbc05ae7bb1607a5719b701b40383 2024-11-16T14:57:07,485 DEBUG [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegionServer(2266): Finished post open deploy task for testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ggg,1731769026392.dcd6414ddebad66e87ab37a6dba924f3. 2024-11-16T14:57:07,486 INFO [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] handler.AssignRegionHandler(153): Opened testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ggg,1731769026392.dcd6414ddebad66e87ab37a6dba924f3. 2024-11-16T14:57:07,486 INFO [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] handler.AssignRegionHandler(132): Open testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,fff,1731769026392.a5db9431e22152bd2a82e84504d2879a. 2024-11-16T14:57:07,486 INFO [StoreOpener-c2acbc05ae7bb1607a5719b701b40383-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region c2acbc05ae7bb1607a5719b701b40383 columnFamilyName colfamily21 2024-11-16T14:57:07,486 DEBUG [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(7752): Opening region: {ENCODED => a5db9431e22152bd2a82e84504d2879a, NAME => 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,fff,1731769026392.a5db9431e22152bd2a82e84504d2879a.', STARTKEY => 'fff', ENDKEY => 'ggg'} 2024-11-16T14:57:07,486 DEBUG [StoreOpener-c2acbc05ae7bb1607a5719b701b40383-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:57:07,487 DEBUG [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true a5db9431e22152bd2a82e84504d2879a 2024-11-16T14:57:07,487 DEBUG [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(898): Instantiated testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,fff,1731769026392.a5db9431e22152bd2a82e84504d2879a.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-16T14:57:07,487 DEBUG [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(7794): checking encryption for a5db9431e22152bd2a82e84504d2879a 2024-11-16T14:57:07,487 INFO [StoreOpener-c2acbc05ae7bb1607a5719b701b40383-1 {}] regionserver.HStore(327): Store=c2acbc05ae7bb1607a5719b701b40383/colfamily21, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-16T14:57:07,487 DEBUG [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(7797): checking classloading for a5db9431e22152bd2a82e84504d2879a 2024-11-16T14:57:07,487 INFO [StoreOpener-c2acbc05ae7bb1607a5719b701b40383-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily31 of region c2acbc05ae7bb1607a5719b701b40383 2024-11-16T14:57:07,489 INFO [StoreOpener-a5db9431e22152bd2a82e84504d2879a-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily11 of region a5db9431e22152bd2a82e84504d2879a 2024-11-16T14:57:07,489 INFO [StoreOpener-c2acbc05ae7bb1607a5719b701b40383-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region c2acbc05ae7bb1607a5719b701b40383 columnFamilyName colfamily31 2024-11-16T14:57:07,490 DEBUG [StoreOpener-c2acbc05ae7bb1607a5719b701b40383-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:57:07,490 INFO [StoreOpener-c2acbc05ae7bb1607a5719b701b40383-1 {}] regionserver.HStore(327): Store=c2acbc05ae7bb1607a5719b701b40383/colfamily31, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-16T14:57:07,491 DEBUG [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(1038): replaying wal for c2acbc05ae7bb1607a5719b701b40383 2024-11-16T14:57:07,492 INFO [StoreOpener-a5db9431e22152bd2a82e84504d2879a-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region a5db9431e22152bd2a82e84504d2879a columnFamilyName colfamily11 2024-11-16T14:57:07,492 DEBUG [StoreOpener-a5db9431e22152bd2a82e84504d2879a-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:57:07,492 DEBUG [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42223/user/jenkins/test-data/a8176cb1-2ebb-b530-b3c7-53c9e53c186e/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/c2acbc05ae7bb1607a5719b701b40383 2024-11-16T14:57:07,493 INFO [StoreOpener-5bb22144956dcd43c9b1d957232b11e7-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 5bb22144956dcd43c9b1d957232b11e7 columnFamilyName colfamily21 2024-11-16T14:57:07,493 DEBUG [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42223/user/jenkins/test-data/a8176cb1-2ebb-b530-b3c7-53c9e53c186e/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/c2acbc05ae7bb1607a5719b701b40383 2024-11-16T14:57:07,493 DEBUG [StoreOpener-5bb22144956dcd43c9b1d957232b11e7-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:57:07,493 INFO [StoreOpener-a5db9431e22152bd2a82e84504d2879a-1 {}] regionserver.HStore(327): Store=a5db9431e22152bd2a82e84504d2879a/colfamily11, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-16T14:57:07,493 INFO [StoreOpener-a5db9431e22152bd2a82e84504d2879a-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily21 of region a5db9431e22152bd2a82e84504d2879a 2024-11-16T14:57:07,494 INFO [StoreOpener-5bb22144956dcd43c9b1d957232b11e7-1 {}] regionserver.HStore(327): Store=5bb22144956dcd43c9b1d957232b11e7/colfamily21, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-16T14:57:07,494 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=11 updating hbase:meta row=dcd6414ddebad66e87ab37a6dba924f3, regionState=OPEN, openSeqNum=2, regionLocation=fc3fda29f660,42985,1731769023951 2024-11-16T14:57:07,494 INFO [StoreOpener-5bb22144956dcd43c9b1d957232b11e7-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily31 of region 5bb22144956dcd43c9b1d957232b11e7 2024-11-16T14:57:07,495 DEBUG [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(1048): stopping wal replay for c2acbc05ae7bb1607a5719b701b40383 2024-11-16T14:57:07,495 DEBUG [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(1060): Cleaning up temporary data for c2acbc05ae7bb1607a5719b701b40383 2024-11-16T14:57:07,496 INFO [StoreOpener-5bb22144956dcd43c9b1d957232b11e7-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 5bb22144956dcd43c9b1d957232b11e7 columnFamilyName colfamily31 2024-11-16T14:57:07,496 INFO [StoreOpener-a5db9431e22152bd2a82e84504d2879a-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region a5db9431e22152bd2a82e84504d2879a columnFamilyName colfamily21 2024-11-16T14:57:07,497 DEBUG [StoreOpener-5bb22144956dcd43c9b1d957232b11e7-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:57:07,496 DEBUG [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-16T14:57:07,497 DEBUG [StoreOpener-a5db9431e22152bd2a82e84504d2879a-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:57:07,498 INFO [StoreOpener-5bb22144956dcd43c9b1d957232b11e7-1 {}] regionserver.HStore(327): Store=5bb22144956dcd43c9b1d957232b11e7/colfamily31, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-16T14:57:07,498 DEBUG [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(1038): replaying wal for 5bb22144956dcd43c9b1d957232b11e7 2024-11-16T14:57:07,498 INFO [StoreOpener-a5db9431e22152bd2a82e84504d2879a-1 {}] regionserver.HStore(327): Store=a5db9431e22152bd2a82e84504d2879a/colfamily21, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-16T14:57:07,499 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=34, ppid=11, state=RUNNABLE, hasLock=false; OpenRegionProcedure dcd6414ddebad66e87ab37a6dba924f3, server=fc3fda29f660,42985,1731769023951 because future has completed 2024-11-16T14:57:07,499 INFO [StoreOpener-a5db9431e22152bd2a82e84504d2879a-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily31 of region a5db9431e22152bd2a82e84504d2879a 2024-11-16T14:57:07,499 DEBUG [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42223/user/jenkins/test-data/a8176cb1-2ebb-b530-b3c7-53c9e53c186e/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/5bb22144956dcd43c9b1d957232b11e7 2024-11-16T14:57:07,500 DEBUG [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42223/user/jenkins/test-data/a8176cb1-2ebb-b530-b3c7-53c9e53c186e/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/5bb22144956dcd43c9b1d957232b11e7 2024-11-16T14:57:07,501 DEBUG [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(1093): writing seq id for c2acbc05ae7bb1607a5719b701b40383 2024-11-16T14:57:07,502 INFO [StoreOpener-a5db9431e22152bd2a82e84504d2879a-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region a5db9431e22152bd2a82e84504d2879a columnFamilyName colfamily31 2024-11-16T14:57:07,502 DEBUG [StoreOpener-a5db9431e22152bd2a82e84504d2879a-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:57:07,502 DEBUG [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(1048): stopping wal replay for 5bb22144956dcd43c9b1d957232b11e7 2024-11-16T14:57:07,502 DEBUG [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(1060): Cleaning up temporary data for 5bb22144956dcd43c9b1d957232b11e7 2024-11-16T14:57:07,503 DEBUG [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-16T14:57:07,503 INFO [StoreOpener-a5db9431e22152bd2a82e84504d2879a-1 {}] regionserver.HStore(327): Store=a5db9431e22152bd2a82e84504d2879a/colfamily31, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-16T14:57:07,504 DEBUG [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1038): replaying wal for a5db9431e22152bd2a82e84504d2879a 2024-11-16T14:57:07,506 DEBUG [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42223/user/jenkins/test-data/a8176cb1-2ebb-b530-b3c7-53c9e53c186e/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/a5db9431e22152bd2a82e84504d2879a 2024-11-16T14:57:07,507 DEBUG [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(1093): writing seq id for 5bb22144956dcd43c9b1d957232b11e7 2024-11-16T14:57:07,507 DEBUG [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42223/user/jenkins/test-data/a8176cb1-2ebb-b530-b3c7-53c9e53c186e/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/a5db9431e22152bd2a82e84504d2879a 2024-11-16T14:57:07,509 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=34, resume processing ppid=11 2024-11-16T14:57:07,509 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=34, ppid=11, state=SUCCESS, hasLock=false; OpenRegionProcedure dcd6414ddebad66e87ab37a6dba924f3, server=fc3fda29f660,42985,1731769023951 in 267 msec 2024-11-16T14:57:07,511 DEBUG [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1048): stopping wal replay for a5db9431e22152bd2a82e84504d2879a 2024-11-16T14:57:07,511 DEBUG [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1060): Cleaning up temporary data for a5db9431e22152bd2a82e84504d2879a 2024-11-16T14:57:07,512 DEBUG [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-16T14:57:07,513 DEBUG [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:42223/user/jenkins/test-data/a8176cb1-2ebb-b530-b3c7-53c9e53c186e/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/5bb22144956dcd43c9b1d957232b11e7/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-16T14:57:07,513 DEBUG [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:42223/user/jenkins/test-data/a8176cb1-2ebb-b530-b3c7-53c9e53c186e/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/c2acbc05ae7bb1607a5719b701b40383/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-16T14:57:07,513 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=11, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=dcd6414ddebad66e87ab37a6dba924f3, ASSIGN in 471 msec 2024-11-16T14:57:07,514 INFO [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(1114): Opened c2acbc05ae7bb1607a5719b701b40383; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=64073962, jitterRate=-0.04522356390953064}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-16T14:57:07,514 INFO [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(1114): Opened 5bb22144956dcd43c9b1d957232b11e7; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=67946565, jitterRate=0.012482717633247375}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-16T14:57:07,514 DEBUG [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(1122): Running coprocessor post-open hooks for c2acbc05ae7bb1607a5719b701b40383 2024-11-16T14:57:07,514 DEBUG [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 5bb22144956dcd43c9b1d957232b11e7 2024-11-16T14:57:07,514 DEBUG [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(1006): Region open journal for 5bb22144956dcd43c9b1d957232b11e7: Running coprocessor pre-open hook at 1731769027466Writing region info on filesystem at 1731769027466Initializing all the Stores at 1731769027468 (+2 ms)Instantiating store for column family {NAME => 'colfamily11', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731769027468Instantiating store for column family {NAME => 'colfamily21', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731769027474 (+6 ms)Instantiating store for column family {NAME => 'colfamily31', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731769027474Cleaning up temporary data from old regions at 1731769027502 (+28 ms)Running coprocessor post-open hooks at 1731769027514 (+12 ms)Region opened successfully at 1731769027514 2024-11-16T14:57:07,514 DEBUG [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(1006): Region open journal for c2acbc05ae7bb1607a5719b701b40383: Running coprocessor pre-open hook at 1731769027469Writing region info on filesystem at 1731769027469Initializing all the Stores at 1731769027470 (+1 ms)Instantiating store for column family {NAME => 'colfamily11', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731769027470Instantiating store for column family {NAME => 'colfamily21', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731769027476 (+6 ms)Instantiating store for column family {NAME => 'colfamily31', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731769027476Cleaning up temporary data from old regions at 1731769027495 (+19 ms)Running coprocessor post-open hooks at 1731769027514 (+19 ms)Region opened successfully at 1731769027514 2024-11-16T14:57:07,515 DEBUG [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1093): writing seq id for a5db9431e22152bd2a82e84504d2879a 2024-11-16T14:57:07,516 INFO [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegionServer(2236): Post open deploy tasks for testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,mmm,1731769026392.c2acbc05ae7bb1607a5719b701b40383., pid=53, masterSystemTime=1731769027460 2024-11-16T14:57:07,516 INFO [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegionServer(2236): Post open deploy tasks for testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ccc,1731769026392.5bb22144956dcd43c9b1d957232b11e7., pid=47, masterSystemTime=1731769027425 2024-11-16T14:57:07,520 DEBUG [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegionServer(2266): Finished post open deploy task for testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,mmm,1731769026392.c2acbc05ae7bb1607a5719b701b40383. 2024-11-16T14:57:07,520 INFO [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] handler.AssignRegionHandler(153): Opened testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,mmm,1731769026392.c2acbc05ae7bb1607a5719b701b40383. 2024-11-16T14:57:07,520 INFO [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] handler.AssignRegionHandler(132): Open testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,sss,1731769026392.33054df03c1be51dc860405880be6075. 2024-11-16T14:57:07,520 DEBUG [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:42223/user/jenkins/test-data/a8176cb1-2ebb-b530-b3c7-53c9e53c186e/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/a5db9431e22152bd2a82e84504d2879a/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-16T14:57:07,520 DEBUG [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(7752): Opening region: {ENCODED => 33054df03c1be51dc860405880be6075, NAME => 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,sss,1731769026392.33054df03c1be51dc860405880be6075.', STARTKEY => 'sss', ENDKEY => 'ttt'} 2024-11-16T14:57:07,521 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=17 updating hbase:meta row=c2acbc05ae7bb1607a5719b701b40383, regionState=OPEN, openSeqNum=2, regionLocation=fc3fda29f660,38897,1731769023879 2024-11-16T14:57:07,521 DEBUG [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegionServer(2266): Finished post open deploy task for testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ccc,1731769026392.5bb22144956dcd43c9b1d957232b11e7. 2024-11-16T14:57:07,521 INFO [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] handler.AssignRegionHandler(153): Opened testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ccc,1731769026392.5bb22144956dcd43c9b1d957232b11e7. 2024-11-16T14:57:07,521 INFO [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] handler.AssignRegionHandler(132): Open testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,nnn,1731769026392.994ca1526cf6c4838e65f85444853f1a. 2024-11-16T14:57:07,521 DEBUG [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true 33054df03c1be51dc860405880be6075 2024-11-16T14:57:07,522 DEBUG [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(898): Instantiated testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,sss,1731769026392.33054df03c1be51dc860405880be6075.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-16T14:57:07,522 DEBUG [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(7794): checking encryption for 33054df03c1be51dc860405880be6075 2024-11-16T14:57:07,522 DEBUG [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(7752): Opening region: {ENCODED => 994ca1526cf6c4838e65f85444853f1a, NAME => 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,nnn,1731769026392.994ca1526cf6c4838e65f85444853f1a.', STARTKEY => 'nnn', ENDKEY => 'ooo'} 2024-11-16T14:57:07,522 DEBUG [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(7797): checking classloading for 33054df03c1be51dc860405880be6075 2024-11-16T14:57:07,522 DEBUG [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true 994ca1526cf6c4838e65f85444853f1a 2024-11-16T14:57:07,522 DEBUG [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(898): Instantiated testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,nnn,1731769026392.994ca1526cf6c4838e65f85444853f1a.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-16T14:57:07,522 DEBUG [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(7794): checking encryption for 994ca1526cf6c4838e65f85444853f1a 2024-11-16T14:57:07,523 DEBUG [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(7797): checking classloading for 994ca1526cf6c4838e65f85444853f1a 2024-11-16T14:57:07,523 INFO [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1114): Opened a5db9431e22152bd2a82e84504d2879a; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=60559511, jitterRate=-0.09759296476840973}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-16T14:57:07,523 DEBUG [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1122): Running coprocessor post-open hooks for a5db9431e22152bd2a82e84504d2879a 2024-11-16T14:57:07,523 DEBUG [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1006): Region open journal for a5db9431e22152bd2a82e84504d2879a: Running coprocessor pre-open hook at 1731769027487Writing region info on filesystem at 1731769027487Initializing all the Stores at 1731769027489 (+2 ms)Instantiating store for column family {NAME => 'colfamily11', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731769027489Instantiating store for column family {NAME => 'colfamily21', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731769027489Instantiating store for column family {NAME => 'colfamily31', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731769027489Cleaning up temporary data from old regions at 1731769027511 (+22 ms)Running coprocessor post-open hooks at 1731769027523 (+12 ms)Region opened successfully at 1731769027523 2024-11-16T14:57:07,524 INFO [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegionServer(2236): Post open deploy tasks for testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,fff,1731769026392.a5db9431e22152bd2a82e84504d2879a., pid=33, masterSystemTime=1731769027371 2024-11-16T14:57:07,524 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=53, ppid=17, state=RUNNABLE, hasLock=false; OpenRegionProcedure c2acbc05ae7bb1607a5719b701b40383, server=fc3fda29f660,38897,1731769023879 because future has completed 2024-11-16T14:57:07,526 INFO [StoreOpener-33054df03c1be51dc860405880be6075-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily11 of region 33054df03c1be51dc860405880be6075 2024-11-16T14:57:07,526 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=5bb22144956dcd43c9b1d957232b11e7, regionState=OPEN, openSeqNum=2, regionLocation=fc3fda29f660,39089,1731769023769 2024-11-16T14:57:07,528 INFO [StoreOpener-994ca1526cf6c4838e65f85444853f1a-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily11 of region 994ca1526cf6c4838e65f85444853f1a 2024-11-16T14:57:07,529 DEBUG [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegionServer(2266): Finished post open deploy task for testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,fff,1731769026392.a5db9431e22152bd2a82e84504d2879a. 2024-11-16T14:57:07,529 INFO [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] handler.AssignRegionHandler(153): Opened testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,fff,1731769026392.a5db9431e22152bd2a82e84504d2879a. 2024-11-16T14:57:07,529 INFO [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] handler.AssignRegionHandler(132): Open testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,bbb,1731769026392.c3dde5f5f668d914608a2987d4f4c001. 2024-11-16T14:57:07,529 DEBUG [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(7752): Opening region: {ENCODED => c3dde5f5f668d914608a2987d4f4c001, NAME => 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,bbb,1731769026392.c3dde5f5f668d914608a2987d4f4c001.', STARTKEY => 'bbb', ENDKEY => 'ccc'} 2024-11-16T14:57:07,529 INFO [StoreOpener-33054df03c1be51dc860405880be6075-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 33054df03c1be51dc860405880be6075 columnFamilyName colfamily11 2024-11-16T14:57:07,530 DEBUG [StoreOpener-33054df03c1be51dc860405880be6075-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:57:07,530 DEBUG [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true c3dde5f5f668d914608a2987d4f4c001 2024-11-16T14:57:07,531 INFO [StoreOpener-994ca1526cf6c4838e65f85444853f1a-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 994ca1526cf6c4838e65f85444853f1a columnFamilyName colfamily11 2024-11-16T14:57:07,531 DEBUG [StoreOpener-994ca1526cf6c4838e65f85444853f1a-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:57:07,531 INFO [StoreOpener-33054df03c1be51dc860405880be6075-1 {}] regionserver.HStore(327): Store=33054df03c1be51dc860405880be6075/colfamily11, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-16T14:57:07,531 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=47, ppid=7, state=RUNNABLE, hasLock=false; OpenRegionProcedure 5bb22144956dcd43c9b1d957232b11e7, server=fc3fda29f660,39089,1731769023769 because future has completed 2024-11-16T14:57:07,531 INFO [StoreOpener-33054df03c1be51dc860405880be6075-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily21 of region 33054df03c1be51dc860405880be6075 2024-11-16T14:57:07,532 DEBUG [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(898): Instantiated testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,bbb,1731769026392.c3dde5f5f668d914608a2987d4f4c001.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-16T14:57:07,532 INFO [StoreOpener-994ca1526cf6c4838e65f85444853f1a-1 {}] regionserver.HStore(327): Store=994ca1526cf6c4838e65f85444853f1a/colfamily11, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-16T14:57:07,532 INFO [StoreOpener-994ca1526cf6c4838e65f85444853f1a-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily21 of region 994ca1526cf6c4838e65f85444853f1a 2024-11-16T14:57:07,533 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=10 updating hbase:meta row=a5db9431e22152bd2a82e84504d2879a, regionState=OPEN, openSeqNum=2, regionLocation=fc3fda29f660,42985,1731769023951 2024-11-16T14:57:07,534 INFO [StoreOpener-33054df03c1be51dc860405880be6075-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 33054df03c1be51dc860405880be6075 columnFamilyName colfamily21 2024-11-16T14:57:07,534 INFO [StoreOpener-994ca1526cf6c4838e65f85444853f1a-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 994ca1526cf6c4838e65f85444853f1a columnFamilyName colfamily21 2024-11-16T14:57:07,534 DEBUG [StoreOpener-33054df03c1be51dc860405880be6075-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:57:07,534 DEBUG [StoreOpener-994ca1526cf6c4838e65f85444853f1a-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:57:07,532 DEBUG [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(7794): checking encryption for c3dde5f5f668d914608a2987d4f4c001 2024-11-16T14:57:07,534 DEBUG [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(7797): checking classloading for c3dde5f5f668d914608a2987d4f4c001 2024-11-16T14:57:07,535 INFO [StoreOpener-994ca1526cf6c4838e65f85444853f1a-1 {}] regionserver.HStore(327): Store=994ca1526cf6c4838e65f85444853f1a/colfamily21, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-16T14:57:07,535 INFO [StoreOpener-33054df03c1be51dc860405880be6075-1 {}] regionserver.HStore(327): Store=33054df03c1be51dc860405880be6075/colfamily21, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-16T14:57:07,535 INFO [StoreOpener-33054df03c1be51dc860405880be6075-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily31 of region 33054df03c1be51dc860405880be6075 2024-11-16T14:57:07,536 INFO [StoreOpener-994ca1526cf6c4838e65f85444853f1a-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily31 of region 994ca1526cf6c4838e65f85444853f1a 2024-11-16T14:57:07,537 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=33, ppid=10, state=RUNNABLE, hasLock=false; OpenRegionProcedure a5db9431e22152bd2a82e84504d2879a, server=fc3fda29f660,42985,1731769023951 because future has completed 2024-11-16T14:57:07,538 INFO [StoreOpener-33054df03c1be51dc860405880be6075-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 33054df03c1be51dc860405880be6075 columnFamilyName colfamily31 2024-11-16T14:57:07,538 DEBUG [StoreOpener-33054df03c1be51dc860405880be6075-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:57:07,538 INFO [StoreOpener-994ca1526cf6c4838e65f85444853f1a-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 994ca1526cf6c4838e65f85444853f1a columnFamilyName colfamily31 2024-11-16T14:57:07,538 DEBUG [StoreOpener-994ca1526cf6c4838e65f85444853f1a-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:57:07,539 INFO [StoreOpener-994ca1526cf6c4838e65f85444853f1a-1 {}] regionserver.HStore(327): Store=994ca1526cf6c4838e65f85444853f1a/colfamily31, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-16T14:57:07,539 INFO [StoreOpener-33054df03c1be51dc860405880be6075-1 {}] regionserver.HStore(327): Store=33054df03c1be51dc860405880be6075/colfamily31, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-16T14:57:07,539 INFO [StoreOpener-c3dde5f5f668d914608a2987d4f4c001-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily11 of region c3dde5f5f668d914608a2987d4f4c001 2024-11-16T14:57:07,539 DEBUG [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(1038): replaying wal for 994ca1526cf6c4838e65f85444853f1a 2024-11-16T14:57:07,541 DEBUG [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(1038): replaying wal for 33054df03c1be51dc860405880be6075 2024-11-16T14:57:07,541 DEBUG [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42223/user/jenkins/test-data/a8176cb1-2ebb-b530-b3c7-53c9e53c186e/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/994ca1526cf6c4838e65f85444853f1a 2024-11-16T14:57:07,542 DEBUG [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42223/user/jenkins/test-data/a8176cb1-2ebb-b530-b3c7-53c9e53c186e/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/33054df03c1be51dc860405880be6075 2024-11-16T14:57:07,542 DEBUG [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42223/user/jenkins/test-data/a8176cb1-2ebb-b530-b3c7-53c9e53c186e/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/994ca1526cf6c4838e65f85444853f1a 2024-11-16T14:57:07,543 INFO [StoreOpener-c3dde5f5f668d914608a2987d4f4c001-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region c3dde5f5f668d914608a2987d4f4c001 columnFamilyName colfamily11 2024-11-16T14:57:07,543 DEBUG [StoreOpener-c3dde5f5f668d914608a2987d4f4c001-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:57:07,544 DEBUG [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42223/user/jenkins/test-data/a8176cb1-2ebb-b530-b3c7-53c9e53c186e/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/33054df03c1be51dc860405880be6075 2024-11-16T14:57:07,545 INFO [StoreOpener-c3dde5f5f668d914608a2987d4f4c001-1 {}] regionserver.HStore(327): Store=c3dde5f5f668d914608a2987d4f4c001/colfamily11, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-16T14:57:07,545 INFO [StoreOpener-c3dde5f5f668d914608a2987d4f4c001-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily21 of region c3dde5f5f668d914608a2987d4f4c001 2024-11-16T14:57:07,545 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=43615 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=fc3fda29f660,42985,1731769023951, table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=a5db9431e22152bd2a82e84504d2879a. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-16T14:57:07,546 DEBUG [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(1048): stopping wal replay for 33054df03c1be51dc860405880be6075 2024-11-16T14:57:07,546 DEBUG [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(1060): Cleaning up temporary data for 33054df03c1be51dc860405880be6075 2024-11-16T14:57:07,546 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=47, resume processing ppid=7 2024-11-16T14:57:07,546 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=47, ppid=7, state=SUCCESS, hasLock=false; OpenRegionProcedure 5bb22144956dcd43c9b1d957232b11e7, server=fc3fda29f660,39089,1731769023769 in 227 msec 2024-11-16T14:57:07,547 DEBUG [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(1048): stopping wal replay for 994ca1526cf6c4838e65f85444853f1a 2024-11-16T14:57:07,547 DEBUG [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-16T14:57:07,547 DEBUG [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(1060): Cleaning up temporary data for 994ca1526cf6c4838e65f85444853f1a 2024-11-16T14:57:07,548 DEBUG [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-16T14:57:07,549 INFO [StoreOpener-c3dde5f5f668d914608a2987d4f4c001-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region c3dde5f5f668d914608a2987d4f4c001 columnFamilyName colfamily21 2024-11-16T14:57:07,549 DEBUG [StoreOpener-c3dde5f5f668d914608a2987d4f4c001-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:57:07,550 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=7, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=5bb22144956dcd43c9b1d957232b11e7, ASSIGN in 508 msec 2024-11-16T14:57:07,550 DEBUG [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(1093): writing seq id for 33054df03c1be51dc860405880be6075 2024-11-16T14:57:07,550 DEBUG [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(1093): writing seq id for 994ca1526cf6c4838e65f85444853f1a 2024-11-16T14:57:07,551 INFO [StoreOpener-c3dde5f5f668d914608a2987d4f4c001-1 {}] regionserver.HStore(327): Store=c3dde5f5f668d914608a2987d4f4c001/colfamily21, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-16T14:57:07,551 INFO [StoreOpener-c3dde5f5f668d914608a2987d4f4c001-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily31 of region c3dde5f5f668d914608a2987d4f4c001 2024-11-16T14:57:07,554 INFO [StoreOpener-c3dde5f5f668d914608a2987d4f4c001-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region c3dde5f5f668d914608a2987d4f4c001 columnFamilyName colfamily31 2024-11-16T14:57:07,554 DEBUG [StoreOpener-c3dde5f5f668d914608a2987d4f4c001-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:57:07,555 INFO [StoreOpener-c3dde5f5f668d914608a2987d4f4c001-1 {}] regionserver.HStore(327): Store=c3dde5f5f668d914608a2987d4f4c001/colfamily31, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-16T14:57:07,555 DEBUG [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1038): replaying wal for c3dde5f5f668d914608a2987d4f4c001 2024-11-16T14:57:07,557 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=33, resume processing ppid=10 2024-11-16T14:57:07,557 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=33, ppid=10, state=SUCCESS, hasLock=false; OpenRegionProcedure a5db9431e22152bd2a82e84504d2879a, server=fc3fda29f660,42985,1731769023951 in 321 msec 2024-11-16T14:57:07,557 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=43615 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=fc3fda29f660,38897,1731769023879, table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=c2acbc05ae7bb1607a5719b701b40383. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-16T14:57:07,558 DEBUG [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42223/user/jenkins/test-data/a8176cb1-2ebb-b530-b3c7-53c9e53c186e/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/c3dde5f5f668d914608a2987d4f4c001 2024-11-16T14:57:07,558 DEBUG [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:42223/user/jenkins/test-data/a8176cb1-2ebb-b530-b3c7-53c9e53c186e/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/33054df03c1be51dc860405880be6075/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-16T14:57:07,559 DEBUG [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42223/user/jenkins/test-data/a8176cb1-2ebb-b530-b3c7-53c9e53c186e/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/c3dde5f5f668d914608a2987d4f4c001 2024-11-16T14:57:07,560 INFO [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(1114): Opened 33054df03c1be51dc860405880be6075; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=59087551, jitterRate=-0.11952687799930573}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-16T14:57:07,560 DEBUG [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 33054df03c1be51dc860405880be6075 2024-11-16T14:57:07,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43615 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-16T14:57:07,560 DEBUG [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(1006): Region open journal for 33054df03c1be51dc860405880be6075: Running coprocessor pre-open hook at 1731769027522Writing region info on filesystem at 1731769027522Initializing all the Stores at 1731769027525 (+3 ms)Instantiating store for column family {NAME => 'colfamily11', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731769027525Instantiating store for column family {NAME => 'colfamily21', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731769027526 (+1 ms)Instantiating store for column family {NAME => 'colfamily31', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731769027526Cleaning up temporary data from old regions at 1731769027546 (+20 ms)Running coprocessor post-open hooks at 1731769027560 (+14 ms)Region opened successfully at 1731769027560 2024-11-16T14:57:07,560 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=10, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=a5db9431e22152bd2a82e84504d2879a, ASSIGN in 518 msec 2024-11-16T14:57:07,561 DEBUG [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1048): stopping wal replay for c3dde5f5f668d914608a2987d4f4c001 2024-11-16T14:57:07,561 DEBUG [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1060): Cleaning up temporary data for c3dde5f5f668d914608a2987d4f4c001 2024-11-16T14:57:07,561 INFO [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegionServer(2236): Post open deploy tasks for testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,sss,1731769026392.33054df03c1be51dc860405880be6075., pid=55, masterSystemTime=1731769027460 2024-11-16T14:57:07,562 DEBUG [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-16T14:57:07,564 DEBUG [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:42223/user/jenkins/test-data/a8176cb1-2ebb-b530-b3c7-53c9e53c186e/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/994ca1526cf6c4838e65f85444853f1a/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-16T14:57:07,565 DEBUG [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegionServer(2266): Finished post open deploy task for testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,sss,1731769026392.33054df03c1be51dc860405880be6075. 2024-11-16T14:57:07,565 INFO [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] handler.AssignRegionHandler(153): Opened testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,sss,1731769026392.33054df03c1be51dc860405880be6075. 2024-11-16T14:57:07,565 INFO [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] handler.AssignRegionHandler(132): Open testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,iii,1731769026392.7352dcdbf9811dcebc51a714a1de4f55. 2024-11-16T14:57:07,566 DEBUG [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(7752): Opening region: {ENCODED => 7352dcdbf9811dcebc51a714a1de4f55, NAME => 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,iii,1731769026392.7352dcdbf9811dcebc51a714a1de4f55.', STARTKEY => 'iii', ENDKEY => 'jjj'} 2024-11-16T14:57:07,566 DEBUG [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1093): writing seq id for c3dde5f5f668d914608a2987d4f4c001 2024-11-16T14:57:07,566 DEBUG [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true 7352dcdbf9811dcebc51a714a1de4f55 2024-11-16T14:57:07,566 INFO [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(1114): Opened 994ca1526cf6c4838e65f85444853f1a; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=69015434, jitterRate=0.028410106897354126}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-16T14:57:07,566 DEBUG [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 994ca1526cf6c4838e65f85444853f1a 2024-11-16T14:57:07,566 DEBUG [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(898): Instantiated testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,iii,1731769026392.7352dcdbf9811dcebc51a714a1de4f55.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-16T14:57:07,567 DEBUG [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(1006): Region open journal for 994ca1526cf6c4838e65f85444853f1a: Running coprocessor pre-open hook at 1731769027523Writing region info on filesystem at 1731769027523Initializing all the Stores at 1731769027525 (+2 ms)Instantiating store for column family {NAME => 'colfamily11', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731769027526 (+1 ms)Instantiating store for column family {NAME => 'colfamily21', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731769027528 (+2 ms)Instantiating store for column family {NAME => 'colfamily31', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731769027528Cleaning up temporary data from old regions at 1731769027547 (+19 ms)Running coprocessor post-open hooks at 1731769027566 (+19 ms)Region opened successfully at 1731769027566 2024-11-16T14:57:07,567 DEBUG [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(7794): checking encryption for 7352dcdbf9811dcebc51a714a1de4f55 2024-11-16T14:57:07,567 DEBUG [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(7797): checking classloading for 7352dcdbf9811dcebc51a714a1de4f55 2024-11-16T14:57:07,568 INFO [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegionServer(2236): Post open deploy tasks for testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,nnn,1731769026392.994ca1526cf6c4838e65f85444853f1a., pid=41, masterSystemTime=1731769027425 2024-11-16T14:57:07,568 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=53, resume processing ppid=17 2024-11-16T14:57:07,568 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=53, ppid=17, state=SUCCESS, hasLock=false; OpenRegionProcedure c2acbc05ae7bb1607a5719b701b40383, server=fc3fda29f660,38897,1731769023879 in 221 msec 2024-11-16T14:57:07,569 INFO [StoreOpener-7352dcdbf9811dcebc51a714a1de4f55-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily11 of region 7352dcdbf9811dcebc51a714a1de4f55 2024-11-16T14:57:07,571 DEBUG [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:42223/user/jenkins/test-data/a8176cb1-2ebb-b530-b3c7-53c9e53c186e/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/c3dde5f5f668d914608a2987d4f4c001/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-16T14:57:07,572 INFO [StoreOpener-7352dcdbf9811dcebc51a714a1de4f55-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 7352dcdbf9811dcebc51a714a1de4f55 columnFamilyName colfamily11 2024-11-16T14:57:07,572 DEBUG [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegionServer(2266): Finished post open deploy task for testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,nnn,1731769026392.994ca1526cf6c4838e65f85444853f1a. 2024-11-16T14:57:07,572 INFO [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] handler.AssignRegionHandler(153): Opened testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,nnn,1731769026392.994ca1526cf6c4838e65f85444853f1a. 2024-11-16T14:57:07,572 INFO [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] handler.AssignRegionHandler(132): Open testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,zzz,1731769026392.2f41a8c996224c818e917389bd3f5141. 2024-11-16T14:57:07,572 INFO [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1114): Opened c3dde5f5f668d914608a2987d4f4c001; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=70246264, jitterRate=0.04675090312957764}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-16T14:57:07,573 DEBUG [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(7752): Opening region: {ENCODED => 2f41a8c996224c818e917389bd3f5141, NAME => 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,zzz,1731769026392.2f41a8c996224c818e917389bd3f5141.', STARTKEY => 'zzz', ENDKEY => ''} 2024-11-16T14:57:07,573 DEBUG [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1122): Running coprocessor post-open hooks for c3dde5f5f668d914608a2987d4f4c001 2024-11-16T14:57:07,573 DEBUG [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1006): Region open journal for c3dde5f5f668d914608a2987d4f4c001: Running coprocessor pre-open hook at 1731769027535Writing region info on filesystem at 1731769027535Initializing all the Stores at 1731769027538 (+3 ms)Instantiating store for column family {NAME => 'colfamily11', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731769027538Instantiating store for column family {NAME => 'colfamily21', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731769027540 (+2 ms)Instantiating store for column family {NAME => 'colfamily31', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731769027540Cleaning up temporary data from old regions at 1731769027561 (+21 ms)Running coprocessor post-open hooks at 1731769027573 (+12 ms)Region opened successfully at 1731769027573 2024-11-16T14:57:07,573 DEBUG [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true 2f41a8c996224c818e917389bd3f5141 2024-11-16T14:57:07,573 DEBUG [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(898): Instantiated testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,zzz,1731769026392.2f41a8c996224c818e917389bd3f5141.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-16T14:57:07,573 DEBUG [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(7794): checking encryption for 2f41a8c996224c818e917389bd3f5141 2024-11-16T14:57:07,573 DEBUG [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(7797): checking classloading for 2f41a8c996224c818e917389bd3f5141 2024-11-16T14:57:07,574 DEBUG [StoreOpener-7352dcdbf9811dcebc51a714a1de4f55-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:57:07,574 INFO [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegionServer(2236): Post open deploy tasks for testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,bbb,1731769026392.c3dde5f5f668d914608a2987d4f4c001., pid=39, masterSystemTime=1731769027371 2024-11-16T14:57:07,575 INFO [StoreOpener-7352dcdbf9811dcebc51a714a1de4f55-1 {}] regionserver.HStore(327): Store=7352dcdbf9811dcebc51a714a1de4f55/colfamily11, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-16T14:57:07,575 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=17, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=c2acbc05ae7bb1607a5719b701b40383, ASSIGN in 529 msec 2024-11-16T14:57:07,575 INFO [StoreOpener-7352dcdbf9811dcebc51a714a1de4f55-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily21 of region 7352dcdbf9811dcebc51a714a1de4f55 2024-11-16T14:57:07,576 INFO [StoreOpener-2f41a8c996224c818e917389bd3f5141-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily11 of region 2f41a8c996224c818e917389bd3f5141 2024-11-16T14:57:07,578 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=18 updating hbase:meta row=994ca1526cf6c4838e65f85444853f1a, regionState=OPEN, openSeqNum=2, regionLocation=fc3fda29f660,39089,1731769023769 2024-11-16T14:57:07,578 INFO [StoreOpener-7352dcdbf9811dcebc51a714a1de4f55-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 7352dcdbf9811dcebc51a714a1de4f55 columnFamilyName colfamily21 2024-11-16T14:57:07,578 DEBUG [StoreOpener-7352dcdbf9811dcebc51a714a1de4f55-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:57:07,579 INFO [StoreOpener-2f41a8c996224c818e917389bd3f5141-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 2f41a8c996224c818e917389bd3f5141 columnFamilyName colfamily11 2024-11-16T14:57:07,579 DEBUG [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegionServer(2266): Finished post open deploy task for testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,bbb,1731769026392.c3dde5f5f668d914608a2987d4f4c001. 2024-11-16T14:57:07,579 DEBUG [StoreOpener-2f41a8c996224c818e917389bd3f5141-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:57:07,579 INFO [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] handler.AssignRegionHandler(153): Opened testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,bbb,1731769026392.c3dde5f5f668d914608a2987d4f4c001. 2024-11-16T14:57:07,579 INFO [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] handler.AssignRegionHandler(132): Open testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,uuu,1731769026392.35ef9a7f35bb0b08ec1d53b82d3f9777. 2024-11-16T14:57:07,579 INFO [StoreOpener-7352dcdbf9811dcebc51a714a1de4f55-1 {}] regionserver.HStore(327): Store=7352dcdbf9811dcebc51a714a1de4f55/colfamily21, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-16T14:57:07,579 INFO [StoreOpener-7352dcdbf9811dcebc51a714a1de4f55-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily31 of region 7352dcdbf9811dcebc51a714a1de4f55 2024-11-16T14:57:07,579 DEBUG [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(7752): Opening region: {ENCODED => 35ef9a7f35bb0b08ec1d53b82d3f9777, NAME => 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,uuu,1731769026392.35ef9a7f35bb0b08ec1d53b82d3f9777.', STARTKEY => 'uuu', ENDKEY => 'vvv'} 2024-11-16T14:57:07,580 DEBUG [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true 35ef9a7f35bb0b08ec1d53b82d3f9777 2024-11-16T14:57:07,580 DEBUG [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(898): Instantiated testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,uuu,1731769026392.35ef9a7f35bb0b08ec1d53b82d3f9777.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-16T14:57:07,580 DEBUG [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(7794): checking encryption for 35ef9a7f35bb0b08ec1d53b82d3f9777 2024-11-16T14:57:07,580 DEBUG [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(7797): checking classloading for 35ef9a7f35bb0b08ec1d53b82d3f9777 2024-11-16T14:57:07,580 INFO [StoreOpener-2f41a8c996224c818e917389bd3f5141-1 {}] regionserver.HStore(327): Store=2f41a8c996224c818e917389bd3f5141/colfamily11, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-16T14:57:07,581 INFO [StoreOpener-2f41a8c996224c818e917389bd3f5141-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily21 of region 2f41a8c996224c818e917389bd3f5141 2024-11-16T14:57:07,582 INFO [StoreOpener-7352dcdbf9811dcebc51a714a1de4f55-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 7352dcdbf9811dcebc51a714a1de4f55 columnFamilyName colfamily31 2024-11-16T14:57:07,582 DEBUG [StoreOpener-7352dcdbf9811dcebc51a714a1de4f55-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:57:07,582 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=41, ppid=18, state=RUNNABLE, hasLock=false; OpenRegionProcedure 994ca1526cf6c4838e65f85444853f1a, server=fc3fda29f660,39089,1731769023769 because future has completed 2024-11-16T14:57:07,583 INFO [StoreOpener-7352dcdbf9811dcebc51a714a1de4f55-1 {}] regionserver.HStore(327): Store=7352dcdbf9811dcebc51a714a1de4f55/colfamily31, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-16T14:57:07,583 DEBUG [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(1038): replaying wal for 7352dcdbf9811dcebc51a714a1de4f55 2024-11-16T14:57:07,583 INFO [StoreOpener-2f41a8c996224c818e917389bd3f5141-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 2f41a8c996224c818e917389bd3f5141 columnFamilyName colfamily21 2024-11-16T14:57:07,584 DEBUG [StoreOpener-2f41a8c996224c818e917389bd3f5141-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:57:07,584 DEBUG [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42223/user/jenkins/test-data/a8176cb1-2ebb-b530-b3c7-53c9e53c186e/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/7352dcdbf9811dcebc51a714a1de4f55 2024-11-16T14:57:07,584 INFO [StoreOpener-2f41a8c996224c818e917389bd3f5141-1 {}] regionserver.HStore(327): Store=2f41a8c996224c818e917389bd3f5141/colfamily21, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-16T14:57:07,585 INFO [StoreOpener-2f41a8c996224c818e917389bd3f5141-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily31 of region 2f41a8c996224c818e917389bd3f5141 2024-11-16T14:57:07,585 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=6 updating hbase:meta row=c3dde5f5f668d914608a2987d4f4c001, regionState=OPEN, openSeqNum=2, regionLocation=fc3fda29f660,42985,1731769023951 2024-11-16T14:57:07,585 DEBUG [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42223/user/jenkins/test-data/a8176cb1-2ebb-b530-b3c7-53c9e53c186e/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/7352dcdbf9811dcebc51a714a1de4f55 2024-11-16T14:57:07,587 INFO [StoreOpener-2f41a8c996224c818e917389bd3f5141-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 2f41a8c996224c818e917389bd3f5141 columnFamilyName colfamily31 2024-11-16T14:57:07,587 DEBUG [StoreOpener-2f41a8c996224c818e917389bd3f5141-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:57:07,588 DEBUG [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(1048): stopping wal replay for 7352dcdbf9811dcebc51a714a1de4f55 2024-11-16T14:57:07,588 INFO [StoreOpener-2f41a8c996224c818e917389bd3f5141-1 {}] regionserver.HStore(327): Store=2f41a8c996224c818e917389bd3f5141/colfamily31, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-16T14:57:07,589 DEBUG [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(1038): replaying wal for 2f41a8c996224c818e917389bd3f5141 2024-11-16T14:57:07,589 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=39, ppid=6, state=RUNNABLE, hasLock=false; OpenRegionProcedure c3dde5f5f668d914608a2987d4f4c001, server=fc3fda29f660,42985,1731769023951 because future has completed 2024-11-16T14:57:07,589 DEBUG [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(1060): Cleaning up temporary data for 7352dcdbf9811dcebc51a714a1de4f55 2024-11-16T14:57:07,590 DEBUG [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-16T14:57:07,590 DEBUG [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42223/user/jenkins/test-data/a8176cb1-2ebb-b530-b3c7-53c9e53c186e/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/2f41a8c996224c818e917389bd3f5141 2024-11-16T14:57:07,591 DEBUG [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42223/user/jenkins/test-data/a8176cb1-2ebb-b530-b3c7-53c9e53c186e/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/2f41a8c996224c818e917389bd3f5141 2024-11-16T14:57:07,594 DEBUG [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(1093): writing seq id for 7352dcdbf9811dcebc51a714a1de4f55 2024-11-16T14:57:07,594 DEBUG [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(1048): stopping wal replay for 2f41a8c996224c818e917389bd3f5141 2024-11-16T14:57:07,594 DEBUG [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(1060): Cleaning up temporary data for 2f41a8c996224c818e917389bd3f5141 2024-11-16T14:57:07,595 DEBUG [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-16T14:57:07,598 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=41, resume processing ppid=18 2024-11-16T14:57:07,598 DEBUG [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(1093): writing seq id for 2f41a8c996224c818e917389bd3f5141 2024-11-16T14:57:07,598 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=41, ppid=18, state=SUCCESS, hasLock=false; OpenRegionProcedure 994ca1526cf6c4838e65f85444853f1a, server=fc3fda29f660,39089,1731769023769 in 315 msec 2024-11-16T14:57:07,599 INFO [StoreOpener-35ef9a7f35bb0b08ec1d53b82d3f9777-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily11 of region 35ef9a7f35bb0b08ec1d53b82d3f9777 2024-11-16T14:57:07,600 DEBUG [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:42223/user/jenkins/test-data/a8176cb1-2ebb-b530-b3c7-53c9e53c186e/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/7352dcdbf9811dcebc51a714a1de4f55/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-16T14:57:07,601 INFO [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(1114): Opened 7352dcdbf9811dcebc51a714a1de4f55; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=65955829, jitterRate=-0.017181560397148132}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-16T14:57:07,601 DEBUG [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 7352dcdbf9811dcebc51a714a1de4f55 2024-11-16T14:57:07,601 DEBUG [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(1006): Region open journal for 7352dcdbf9811dcebc51a714a1de4f55: Running coprocessor pre-open hook at 1731769027567Writing region info on filesystem at 1731769027567Initializing all the Stores at 1731769027569 (+2 ms)Instantiating store for column family {NAME => 'colfamily11', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731769027569Instantiating store for column family {NAME => 'colfamily21', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731769027569Instantiating store for column family {NAME => 'colfamily31', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731769027569Cleaning up temporary data from old regions at 1731769027590 (+21 ms)Running coprocessor post-open hooks at 1731769027601 (+11 ms)Region opened successfully at 1731769027601 2024-11-16T14:57:07,601 INFO [StoreOpener-35ef9a7f35bb0b08ec1d53b82d3f9777-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 35ef9a7f35bb0b08ec1d53b82d3f9777 columnFamilyName colfamily11 2024-11-16T14:57:07,602 DEBUG [StoreOpener-35ef9a7f35bb0b08ec1d53b82d3f9777-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:57:07,602 INFO [StoreOpener-35ef9a7f35bb0b08ec1d53b82d3f9777-1 {}] regionserver.HStore(327): Store=35ef9a7f35bb0b08ec1d53b82d3f9777/colfamily11, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-16T14:57:07,603 INFO [StoreOpener-35ef9a7f35bb0b08ec1d53b82d3f9777-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily21 of region 35ef9a7f35bb0b08ec1d53b82d3f9777 2024-11-16T14:57:07,603 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=18, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=994ca1526cf6c4838e65f85444853f1a, ASSIGN in 560 msec 2024-11-16T14:57:07,603 DEBUG [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:42223/user/jenkins/test-data/a8176cb1-2ebb-b530-b3c7-53c9e53c186e/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/2f41a8c996224c818e917389bd3f5141/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-16T14:57:07,603 INFO [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegionServer(2236): Post open deploy tasks for testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,iii,1731769026392.7352dcdbf9811dcebc51a714a1de4f55., pid=52, masterSystemTime=1731769027460 2024-11-16T14:57:07,605 INFO [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(1114): Opened 2f41a8c996224c818e917389bd3f5141; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=68802891, jitterRate=0.025242969393730164}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-16T14:57:07,605 DEBUG [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 2f41a8c996224c818e917389bd3f5141 2024-11-16T14:57:07,605 DEBUG [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(1006): Region open journal for 2f41a8c996224c818e917389bd3f5141: Running coprocessor pre-open hook at 1731769027574Writing region info on filesystem at 1731769027574Initializing all the Stores at 1731769027575 (+1 ms)Instantiating store for column family {NAME => 'colfamily11', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731769027575Instantiating store for column family {NAME => 'colfamily21', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731769027576 (+1 ms)Instantiating store for column family {NAME => 'colfamily31', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731769027576Cleaning up temporary data from old regions at 1731769027594 (+18 ms)Running coprocessor post-open hooks at 1731769027605 (+11 ms)Region opened successfully at 1731769027605 2024-11-16T14:57:07,606 INFO [StoreOpener-35ef9a7f35bb0b08ec1d53b82d3f9777-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 35ef9a7f35bb0b08ec1d53b82d3f9777 columnFamilyName colfamily21 2024-11-16T14:57:07,606 DEBUG [StoreOpener-35ef9a7f35bb0b08ec1d53b82d3f9777-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:57:07,606 INFO [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegionServer(2236): Post open deploy tasks for testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,zzz,1731769026392.2f41a8c996224c818e917389bd3f5141., pid=44, masterSystemTime=1731769027425 2024-11-16T14:57:07,607 INFO [StoreOpener-35ef9a7f35bb0b08ec1d53b82d3f9777-1 {}] regionserver.HStore(327): Store=35ef9a7f35bb0b08ec1d53b82d3f9777/colfamily21, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-16T14:57:07,607 DEBUG [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegionServer(2266): Finished post open deploy task for testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,iii,1731769026392.7352dcdbf9811dcebc51a714a1de4f55. 2024-11-16T14:57:07,607 INFO [StoreOpener-35ef9a7f35bb0b08ec1d53b82d3f9777-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily31 of region 35ef9a7f35bb0b08ec1d53b82d3f9777 2024-11-16T14:57:07,607 INFO [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] handler.AssignRegionHandler(153): Opened testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,iii,1731769026392.7352dcdbf9811dcebc51a714a1de4f55. 2024-11-16T14:57:07,608 INFO [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] handler.AssignRegionHandler(132): Open testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ppp,1731769026392.7f5c9df3a544933227f58b052f743f72. 2024-11-16T14:57:07,608 DEBUG [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(7752): Opening region: {ENCODED => 7f5c9df3a544933227f58b052f743f72, NAME => 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ppp,1731769026392.7f5c9df3a544933227f58b052f743f72.', STARTKEY => 'ppp', ENDKEY => 'qqq'} 2024-11-16T14:57:07,608 DEBUG [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true 7f5c9df3a544933227f58b052f743f72 2024-11-16T14:57:07,608 DEBUG [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(898): Instantiated testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ppp,1731769026392.7f5c9df3a544933227f58b052f743f72.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-16T14:57:07,609 DEBUG [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(7794): checking encryption for 7f5c9df3a544933227f58b052f743f72 2024-11-16T14:57:07,609 DEBUG [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(7797): checking classloading for 7f5c9df3a544933227f58b052f743f72 2024-11-16T14:57:07,611 INFO [StoreOpener-35ef9a7f35bb0b08ec1d53b82d3f9777-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 35ef9a7f35bb0b08ec1d53b82d3f9777 columnFamilyName colfamily31 2024-11-16T14:57:07,611 DEBUG [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegionServer(2266): Finished post open deploy task for testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,zzz,1731769026392.2f41a8c996224c818e917389bd3f5141. 2024-11-16T14:57:07,611 DEBUG [StoreOpener-35ef9a7f35bb0b08ec1d53b82d3f9777-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:57:07,611 INFO [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] handler.AssignRegionHandler(153): Opened testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,zzz,1731769026392.2f41a8c996224c818e917389bd3f5141. 2024-11-16T14:57:07,611 INFO [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] handler.AssignRegionHandler(132): Open testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ttt,1731769026392.7f11f79823720d664105d8643dfa862d. 2024-11-16T14:57:07,611 DEBUG [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(7752): Opening region: {ENCODED => 7f11f79823720d664105d8643dfa862d, NAME => 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ttt,1731769026392.7f11f79823720d664105d8643dfa862d.', STARTKEY => 'ttt', ENDKEY => 'uuu'} 2024-11-16T14:57:07,611 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=39, resume processing ppid=6 2024-11-16T14:57:07,611 INFO [StoreOpener-35ef9a7f35bb0b08ec1d53b82d3f9777-1 {}] regionserver.HStore(327): Store=35ef9a7f35bb0b08ec1d53b82d3f9777/colfamily31, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-16T14:57:07,611 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=39, ppid=6, state=SUCCESS, hasLock=false; OpenRegionProcedure c3dde5f5f668d914608a2987d4f4c001, server=fc3fda29f660,42985,1731769023951 in 340 msec 2024-11-16T14:57:07,612 DEBUG [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(1038): replaying wal for 35ef9a7f35bb0b08ec1d53b82d3f9777 2024-11-16T14:57:07,612 DEBUG [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true 7f11f79823720d664105d8643dfa862d 2024-11-16T14:57:07,612 DEBUG [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(898): Instantiated testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ttt,1731769026392.7f11f79823720d664105d8643dfa862d.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-16T14:57:07,612 DEBUG [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(7794): checking encryption for 7f11f79823720d664105d8643dfa862d 2024-11-16T14:57:07,612 DEBUG [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(7797): checking classloading for 7f11f79823720d664105d8643dfa862d 2024-11-16T14:57:07,613 DEBUG [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42223/user/jenkins/test-data/a8176cb1-2ebb-b530-b3c7-53c9e53c186e/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/35ef9a7f35bb0b08ec1d53b82d3f9777 2024-11-16T14:57:07,613 INFO [StoreOpener-7f5c9df3a544933227f58b052f743f72-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily11 of region 7f5c9df3a544933227f58b052f743f72 2024-11-16T14:57:07,613 DEBUG [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42223/user/jenkins/test-data/a8176cb1-2ebb-b530-b3c7-53c9e53c186e/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/35ef9a7f35bb0b08ec1d53b82d3f9777 2024-11-16T14:57:07,615 DEBUG [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(1048): stopping wal replay for 35ef9a7f35bb0b08ec1d53b82d3f9777 2024-11-16T14:57:07,615 DEBUG [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(1060): Cleaning up temporary data for 35ef9a7f35bb0b08ec1d53b82d3f9777 2024-11-16T14:57:07,615 INFO [StoreOpener-7f11f79823720d664105d8643dfa862d-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily11 of region 7f11f79823720d664105d8643dfa862d 2024-11-16T14:57:07,616 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=c3dde5f5f668d914608a2987d4f4c001, ASSIGN in 573 msec 2024-11-16T14:57:07,616 DEBUG [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-16T14:57:07,618 INFO [StoreOpener-7f11f79823720d664105d8643dfa862d-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 7f11f79823720d664105d8643dfa862d columnFamilyName colfamily11 2024-11-16T14:57:07,618 INFO [StoreOpener-7f5c9df3a544933227f58b052f743f72-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 7f5c9df3a544933227f58b052f743f72 columnFamilyName colfamily11 2024-11-16T14:57:07,618 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=30 updating hbase:meta row=2f41a8c996224c818e917389bd3f5141, regionState=OPEN, openSeqNum=2, regionLocation=fc3fda29f660,39089,1731769023769 2024-11-16T14:57:07,618 DEBUG [StoreOpener-7f5c9df3a544933227f58b052f743f72-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:57:07,618 DEBUG [StoreOpener-7f11f79823720d664105d8643dfa862d-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:57:07,619 INFO [StoreOpener-7f11f79823720d664105d8643dfa862d-1 {}] regionserver.HStore(327): Store=7f11f79823720d664105d8643dfa862d/colfamily11, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-16T14:57:07,619 DEBUG [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(1093): writing seq id for 35ef9a7f35bb0b08ec1d53b82d3f9777 2024-11-16T14:57:07,619 INFO [StoreOpener-7f11f79823720d664105d8643dfa862d-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily21 of region 7f11f79823720d664105d8643dfa862d 2024-11-16T14:57:07,619 INFO [StoreOpener-7f5c9df3a544933227f58b052f743f72-1 {}] regionserver.HStore(327): Store=7f5c9df3a544933227f58b052f743f72/colfamily11, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-16T14:57:07,619 INFO [StoreOpener-7f5c9df3a544933227f58b052f743f72-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily21 of region 7f5c9df3a544933227f58b052f743f72 2024-11-16T14:57:07,622 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=44, ppid=30, state=RUNNABLE, hasLock=false; OpenRegionProcedure 2f41a8c996224c818e917389bd3f5141, server=fc3fda29f660,39089,1731769023769 because future has completed 2024-11-16T14:57:07,622 INFO [StoreOpener-7f5c9df3a544933227f58b052f743f72-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 7f5c9df3a544933227f58b052f743f72 columnFamilyName colfamily21 2024-11-16T14:57:07,623 DEBUG [StoreOpener-7f5c9df3a544933227f58b052f743f72-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:57:07,623 DEBUG [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:42223/user/jenkins/test-data/a8176cb1-2ebb-b530-b3c7-53c9e53c186e/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/35ef9a7f35bb0b08ec1d53b82d3f9777/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-16T14:57:07,624 INFO [StoreOpener-7f5c9df3a544933227f58b052f743f72-1 {}] regionserver.HStore(327): Store=7f5c9df3a544933227f58b052f743f72/colfamily21, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-16T14:57:07,624 INFO [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(1114): Opened 35ef9a7f35bb0b08ec1d53b82d3f9777; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=65630866, jitterRate=-0.022023886442184448}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-16T14:57:07,624 DEBUG [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 35ef9a7f35bb0b08ec1d53b82d3f9777 2024-11-16T14:57:07,624 INFO [StoreOpener-7f5c9df3a544933227f58b052f743f72-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily31 of region 7f5c9df3a544933227f58b052f743f72 2024-11-16T14:57:07,624 INFO [StoreOpener-7f11f79823720d664105d8643dfa862d-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 7f11f79823720d664105d8643dfa862d columnFamilyName colfamily21 2024-11-16T14:57:07,624 DEBUG [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(1006): Region open journal for 35ef9a7f35bb0b08ec1d53b82d3f9777: Running coprocessor pre-open hook at 1731769027580Writing region info on filesystem at 1731769027580Initializing all the Stores at 1731769027582 (+2 ms)Instantiating store for column family {NAME => 'colfamily11', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731769027582Instantiating store for column family {NAME => 'colfamily21', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731769027598 (+16 ms)Instantiating store for column family {NAME => 'colfamily31', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731769027598Cleaning up temporary data from old regions at 1731769027615 (+17 ms)Running coprocessor post-open hooks at 1731769027624 (+9 ms)Region opened successfully at 1731769027624 2024-11-16T14:57:07,624 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=13 updating hbase:meta row=7352dcdbf9811dcebc51a714a1de4f55, regionState=OPEN, openSeqNum=2, regionLocation=fc3fda29f660,38897,1731769023879 2024-11-16T14:57:07,624 DEBUG [StoreOpener-7f11f79823720d664105d8643dfa862d-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:57:07,625 INFO [StoreOpener-7f11f79823720d664105d8643dfa862d-1 {}] regionserver.HStore(327): Store=7f11f79823720d664105d8643dfa862d/colfamily21, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-16T14:57:07,625 INFO [StoreOpener-7f11f79823720d664105d8643dfa862d-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily31 of region 7f11f79823720d664105d8643dfa862d 2024-11-16T14:57:07,626 INFO [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegionServer(2236): Post open deploy tasks for testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,uuu,1731769026392.35ef9a7f35bb0b08ec1d53b82d3f9777., pid=38, masterSystemTime=1731769027371 2024-11-16T14:57:07,627 INFO [StoreOpener-7f5c9df3a544933227f58b052f743f72-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 7f5c9df3a544933227f58b052f743f72 columnFamilyName colfamily31 2024-11-16T14:57:07,627 DEBUG [StoreOpener-7f5c9df3a544933227f58b052f743f72-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:57:07,628 INFO [StoreOpener-7f5c9df3a544933227f58b052f743f72-1 {}] regionserver.HStore(327): Store=7f5c9df3a544933227f58b052f743f72/colfamily31, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-16T14:57:07,629 DEBUG [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(1038): replaying wal for 7f5c9df3a544933227f58b052f743f72 2024-11-16T14:57:07,629 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=52, ppid=13, state=RUNNABLE, hasLock=false; OpenRegionProcedure 7352dcdbf9811dcebc51a714a1de4f55, server=fc3fda29f660,38897,1731769023879 because future has completed 2024-11-16T14:57:07,630 DEBUG [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegionServer(2266): Finished post open deploy task for testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,uuu,1731769026392.35ef9a7f35bb0b08ec1d53b82d3f9777. 2024-11-16T14:57:07,630 INFO [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] handler.AssignRegionHandler(153): Opened testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,uuu,1731769026392.35ef9a7f35bb0b08ec1d53b82d3f9777. 2024-11-16T14:57:07,630 INFO [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] handler.AssignRegionHandler(132): Open testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,xxx,1731769026392.dc4fcaece350cd66f6bf88cfbdff6f15. 2024-11-16T14:57:07,630 DEBUG [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(7752): Opening region: {ENCODED => dc4fcaece350cd66f6bf88cfbdff6f15, NAME => 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,xxx,1731769026392.dc4fcaece350cd66f6bf88cfbdff6f15.', STARTKEY => 'xxx', ENDKEY => 'yyy'} 2024-11-16T14:57:07,630 DEBUG [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42223/user/jenkins/test-data/a8176cb1-2ebb-b530-b3c7-53c9e53c186e/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/7f5c9df3a544933227f58b052f743f72 2024-11-16T14:57:07,631 DEBUG [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true dc4fcaece350cd66f6bf88cfbdff6f15 2024-11-16T14:57:07,631 DEBUG [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(898): Instantiated testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,xxx,1731769026392.dc4fcaece350cd66f6bf88cfbdff6f15.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-16T14:57:07,631 DEBUG [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(7794): checking encryption for dc4fcaece350cd66f6bf88cfbdff6f15 2024-11-16T14:57:07,631 DEBUG [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42223/user/jenkins/test-data/a8176cb1-2ebb-b530-b3c7-53c9e53c186e/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/7f5c9df3a544933227f58b052f743f72 2024-11-16T14:57:07,631 DEBUG [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(7797): checking classloading for dc4fcaece350cd66f6bf88cfbdff6f15 2024-11-16T14:57:07,632 INFO [StoreOpener-7f11f79823720d664105d8643dfa862d-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 7f11f79823720d664105d8643dfa862d columnFamilyName colfamily31 2024-11-16T14:57:07,632 DEBUG [StoreOpener-7f11f79823720d664105d8643dfa862d-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:57:07,633 INFO [StoreOpener-7f11f79823720d664105d8643dfa862d-1 {}] regionserver.HStore(327): Store=7f11f79823720d664105d8643dfa862d/colfamily31, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-16T14:57:07,633 INFO [StoreOpener-dc4fcaece350cd66f6bf88cfbdff6f15-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily11 of region dc4fcaece350cd66f6bf88cfbdff6f15 2024-11-16T14:57:07,633 DEBUG [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(1038): replaying wal for 7f11f79823720d664105d8643dfa862d 2024-11-16T14:57:07,633 DEBUG [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(1048): stopping wal replay for 7f5c9df3a544933227f58b052f743f72 2024-11-16T14:57:07,634 DEBUG [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(1060): Cleaning up temporary data for 7f5c9df3a544933227f58b052f743f72 2024-11-16T14:57:07,635 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=44, resume processing ppid=30 2024-11-16T14:57:07,635 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=44, ppid=30, state=SUCCESS, hasLock=false; OpenRegionProcedure 2f41a8c996224c818e917389bd3f5141, server=fc3fda29f660,39089,1731769023769 in 336 msec 2024-11-16T14:57:07,635 DEBUG [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42223/user/jenkins/test-data/a8176cb1-2ebb-b530-b3c7-53c9e53c186e/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/7f11f79823720d664105d8643dfa862d 2024-11-16T14:57:07,635 DEBUG [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-16T14:57:07,635 DEBUG [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42223/user/jenkins/test-data/a8176cb1-2ebb-b530-b3c7-53c9e53c186e/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/7f11f79823720d664105d8643dfa862d 2024-11-16T14:57:07,637 INFO [StoreOpener-dc4fcaece350cd66f6bf88cfbdff6f15-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region dc4fcaece350cd66f6bf88cfbdff6f15 columnFamilyName colfamily11 2024-11-16T14:57:07,637 DEBUG [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(1048): stopping wal replay for 7f11f79823720d664105d8643dfa862d 2024-11-16T14:57:07,637 DEBUG [StoreOpener-dc4fcaece350cd66f6bf88cfbdff6f15-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:57:07,637 DEBUG [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(1060): Cleaning up temporary data for 7f11f79823720d664105d8643dfa862d 2024-11-16T14:57:07,637 DEBUG [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(1093): writing seq id for 7f5c9df3a544933227f58b052f743f72 2024-11-16T14:57:07,638 DEBUG [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-16T14:57:07,638 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=30, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=2f41a8c996224c818e917389bd3f5141, ASSIGN in 596 msec 2024-11-16T14:57:07,639 INFO [StoreOpener-dc4fcaece350cd66f6bf88cfbdff6f15-1 {}] regionserver.HStore(327): Store=dc4fcaece350cd66f6bf88cfbdff6f15/colfamily11, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-16T14:57:07,639 INFO [StoreOpener-dc4fcaece350cd66f6bf88cfbdff6f15-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily21 of region dc4fcaece350cd66f6bf88cfbdff6f15 2024-11-16T14:57:07,640 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=25 updating hbase:meta row=35ef9a7f35bb0b08ec1d53b82d3f9777, regionState=OPEN, openSeqNum=2, regionLocation=fc3fda29f660,42985,1731769023951 2024-11-16T14:57:07,641 DEBUG [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(1093): writing seq id for 7f11f79823720d664105d8643dfa862d 2024-11-16T14:57:07,643 INFO [StoreOpener-dc4fcaece350cd66f6bf88cfbdff6f15-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region dc4fcaece350cd66f6bf88cfbdff6f15 columnFamilyName colfamily21 2024-11-16T14:57:07,643 DEBUG [StoreOpener-dc4fcaece350cd66f6bf88cfbdff6f15-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:57:07,643 DEBUG [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:42223/user/jenkins/test-data/a8176cb1-2ebb-b530-b3c7-53c9e53c186e/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/7f5c9df3a544933227f58b052f743f72/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-16T14:57:07,644 INFO [StoreOpener-dc4fcaece350cd66f6bf88cfbdff6f15-1 {}] regionserver.HStore(327): Store=dc4fcaece350cd66f6bf88cfbdff6f15/colfamily21, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-16T14:57:07,644 INFO [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(1114): Opened 7f5c9df3a544933227f58b052f743f72; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=71404352, jitterRate=0.06400775909423828}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-16T14:57:07,644 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=38, ppid=25, state=RUNNABLE, hasLock=false; OpenRegionProcedure 35ef9a7f35bb0b08ec1d53b82d3f9777, server=fc3fda29f660,42985,1731769023951 because future has completed 2024-11-16T14:57:07,644 DEBUG [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 7f5c9df3a544933227f58b052f743f72 2024-11-16T14:57:07,644 INFO [StoreOpener-dc4fcaece350cd66f6bf88cfbdff6f15-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily31 of region dc4fcaece350cd66f6bf88cfbdff6f15 2024-11-16T14:57:07,644 DEBUG [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(1006): Region open journal for 7f5c9df3a544933227f58b052f743f72: Running coprocessor pre-open hook at 1731769027609Writing region info on filesystem at 1731769027609Initializing all the Stores at 1731769027611 (+2 ms)Instantiating store for column family {NAME => 'colfamily11', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731769027611Instantiating store for column family {NAME => 'colfamily21', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731769027612 (+1 ms)Instantiating store for column family {NAME => 'colfamily31', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731769027612Cleaning up temporary data from old regions at 1731769027634 (+22 ms)Running coprocessor post-open hooks at 1731769027644 (+10 ms)Region opened successfully at 1731769027644 2024-11-16T14:57:07,646 INFO [StoreOpener-dc4fcaece350cd66f6bf88cfbdff6f15-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region dc4fcaece350cd66f6bf88cfbdff6f15 columnFamilyName colfamily31 2024-11-16T14:57:07,646 INFO [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegionServer(2236): Post open deploy tasks for testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ppp,1731769026392.7f5c9df3a544933227f58b052f743f72., pid=54, masterSystemTime=1731769027460 2024-11-16T14:57:07,647 DEBUG [StoreOpener-dc4fcaece350cd66f6bf88cfbdff6f15-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:57:07,647 INFO [StoreOpener-dc4fcaece350cd66f6bf88cfbdff6f15-1 {}] regionserver.HStore(327): Store=dc4fcaece350cd66f6bf88cfbdff6f15/colfamily31, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-16T14:57:07,648 DEBUG [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(1038): replaying wal for dc4fcaece350cd66f6bf88cfbdff6f15 2024-11-16T14:57:07,649 DEBUG [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42223/user/jenkins/test-data/a8176cb1-2ebb-b530-b3c7-53c9e53c186e/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/dc4fcaece350cd66f6bf88cfbdff6f15 2024-11-16T14:57:07,649 DEBUG [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42223/user/jenkins/test-data/a8176cb1-2ebb-b530-b3c7-53c9e53c186e/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/dc4fcaece350cd66f6bf88cfbdff6f15 2024-11-16T14:57:07,651 DEBUG [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(1048): stopping wal replay for dc4fcaece350cd66f6bf88cfbdff6f15 2024-11-16T14:57:07,651 DEBUG [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(1060): Cleaning up temporary data for dc4fcaece350cd66f6bf88cfbdff6f15 2024-11-16T14:57:07,652 DEBUG [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-16T14:57:07,653 DEBUG [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:42223/user/jenkins/test-data/a8176cb1-2ebb-b530-b3c7-53c9e53c186e/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/7f11f79823720d664105d8643dfa862d/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-16T14:57:07,653 DEBUG [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(1093): writing seq id for dc4fcaece350cd66f6bf88cfbdff6f15 2024-11-16T14:57:07,654 DEBUG [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegionServer(2266): Finished post open deploy task for testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ppp,1731769026392.7f5c9df3a544933227f58b052f743f72. 2024-11-16T14:57:07,654 INFO [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] handler.AssignRegionHandler(153): Opened testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ppp,1731769026392.7f5c9df3a544933227f58b052f743f72. 2024-11-16T14:57:07,654 INFO [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] handler.AssignRegionHandler(132): Open testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,,1731769026392.4f0d635e77a1e3dd04c4d107bb0f1564. 2024-11-16T14:57:07,654 DEBUG [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(7752): Opening region: {ENCODED => 4f0d635e77a1e3dd04c4d107bb0f1564, NAME => 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,,1731769026392.4f0d635e77a1e3dd04c4d107bb0f1564.', STARTKEY => '', ENDKEY => 'bbb'} 2024-11-16T14:57:07,655 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=43615 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=fc3fda29f660,42985,1731769023951, table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=35ef9a7f35bb0b08ec1d53b82d3f9777. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-16T14:57:07,655 DEBUG [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true 4f0d635e77a1e3dd04c4d107bb0f1564 2024-11-16T14:57:07,655 DEBUG [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(898): Instantiated testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,,1731769026392.4f0d635e77a1e3dd04c4d107bb0f1564.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-16T14:57:07,655 INFO [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(1114): Opened 7f11f79823720d664105d8643dfa862d; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=60042269, jitterRate=-0.10530047118663788}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-16T14:57:07,655 DEBUG [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 7f11f79823720d664105d8643dfa862d 2024-11-16T14:57:07,655 DEBUG [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(7794): checking encryption for 4f0d635e77a1e3dd04c4d107bb0f1564 2024-11-16T14:57:07,655 DEBUG [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(7797): checking classloading for 4f0d635e77a1e3dd04c4d107bb0f1564 2024-11-16T14:57:07,655 DEBUG [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(1006): Region open journal for 7f11f79823720d664105d8643dfa862d: Running coprocessor pre-open hook at 1731769027612Writing region info on filesystem at 1731769027612Initializing all the Stores at 1731769027614 (+2 ms)Instantiating store for column family {NAME => 'colfamily11', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731769027614Instantiating store for column family {NAME => 'colfamily21', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731769027614Instantiating store for column family {NAME => 'colfamily31', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731769027614Cleaning up temporary data from old regions at 1731769027637 (+23 ms)Running coprocessor post-open hooks at 1731769027655 (+18 ms)Region opened successfully at 1731769027655 2024-11-16T14:57:07,657 INFO [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegionServer(2236): Post open deploy tasks for testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ttt,1731769026392.7f11f79823720d664105d8643dfa862d., pid=43, masterSystemTime=1731769027425 2024-11-16T14:57:07,657 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=52, resume processing ppid=13 2024-11-16T14:57:07,657 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=52, ppid=13, state=SUCCESS, hasLock=false; OpenRegionProcedure 7352dcdbf9811dcebc51a714a1de4f55, server=fc3fda29f660,38897,1731769023879 in 306 msec 2024-11-16T14:57:07,658 DEBUG [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:42223/user/jenkins/test-data/a8176cb1-2ebb-b530-b3c7-53c9e53c186e/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/dc4fcaece350cd66f6bf88cfbdff6f15/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-16T14:57:07,660 DEBUG [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegionServer(2266): Finished post open deploy task for testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ttt,1731769026392.7f11f79823720d664105d8643dfa862d. 2024-11-16T14:57:07,660 INFO [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] handler.AssignRegionHandler(153): Opened testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ttt,1731769026392.7f11f79823720d664105d8643dfa862d. 2024-11-16T14:57:07,660 INFO [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] handler.AssignRegionHandler(132): Open testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ddd,1731769026392.dde7f54607f11c5fb4271e361e82dabd. 2024-11-16T14:57:07,660 DEBUG [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(7752): Opening region: {ENCODED => dde7f54607f11c5fb4271e361e82dabd, NAME => 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ddd,1731769026392.dde7f54607f11c5fb4271e361e82dabd.', STARTKEY => 'ddd', ENDKEY => 'eee'} 2024-11-16T14:57:07,660 INFO [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(1114): Opened dc4fcaece350cd66f6bf88cfbdff6f15; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=62264857, jitterRate=-0.0721813291311264}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-16T14:57:07,660 DEBUG [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(1122): Running coprocessor post-open hooks for dc4fcaece350cd66f6bf88cfbdff6f15 2024-11-16T14:57:07,661 DEBUG [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(1006): Region open journal for dc4fcaece350cd66f6bf88cfbdff6f15: Running coprocessor pre-open hook at 1731769027631Writing region info on filesystem at 1731769027631Initializing all the Stores at 1731769027633 (+2 ms)Instantiating store for column family {NAME => 'colfamily11', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731769027633Instantiating store for column family {NAME => 'colfamily21', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731769027633Instantiating store for column family {NAME => 'colfamily31', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731769027633Cleaning up temporary data from old regions at 1731769027651 (+18 ms)Running coprocessor post-open hooks at 1731769027660 (+9 ms)Region opened successfully at 1731769027661 (+1 ms) 2024-11-16T14:57:07,661 DEBUG [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true dde7f54607f11c5fb4271e361e82dabd 2024-11-16T14:57:07,661 DEBUG [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(898): Instantiated testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ddd,1731769026392.dde7f54607f11c5fb4271e361e82dabd.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-16T14:57:07,661 DEBUG [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(7794): checking encryption for dde7f54607f11c5fb4271e361e82dabd 2024-11-16T14:57:07,661 DEBUG [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(7797): checking classloading for dde7f54607f11c5fb4271e361e82dabd 2024-11-16T14:57:07,663 INFO [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegionServer(2236): Post open deploy tasks for testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,xxx,1731769026392.dc4fcaece350cd66f6bf88cfbdff6f15., pid=31, masterSystemTime=1731769027371 2024-11-16T14:57:07,663 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=13, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=7352dcdbf9811dcebc51a714a1de4f55, ASSIGN in 619 msec 2024-11-16T14:57:07,663 INFO [StoreOpener-4f0d635e77a1e3dd04c4d107bb0f1564-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily11 of region 4f0d635e77a1e3dd04c4d107bb0f1564 2024-11-16T14:57:07,666 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=24 updating hbase:meta row=7f11f79823720d664105d8643dfa862d, regionState=OPEN, openSeqNum=2, regionLocation=fc3fda29f660,39089,1731769023769 2024-11-16T14:57:07,666 INFO [StoreOpener-4f0d635e77a1e3dd04c4d107bb0f1564-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 4f0d635e77a1e3dd04c4d107bb0f1564 columnFamilyName colfamily11 2024-11-16T14:57:07,666 DEBUG [StoreOpener-4f0d635e77a1e3dd04c4d107bb0f1564-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:57:07,666 DEBUG [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegionServer(2266): Finished post open deploy task for testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,xxx,1731769026392.dc4fcaece350cd66f6bf88cfbdff6f15. 2024-11-16T14:57:07,666 INFO [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] handler.AssignRegionHandler(153): Opened testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,xxx,1731769026392.dc4fcaece350cd66f6bf88cfbdff6f15. 2024-11-16T14:57:07,667 INFO [StoreOpener-dde7f54607f11c5fb4271e361e82dabd-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily11 of region dde7f54607f11c5fb4271e361e82dabd 2024-11-16T14:57:07,667 INFO [StoreOpener-4f0d635e77a1e3dd04c4d107bb0f1564-1 {}] regionserver.HStore(327): Store=4f0d635e77a1e3dd04c4d107bb0f1564/colfamily11, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-16T14:57:07,667 INFO [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] handler.AssignRegionHandler(132): Open testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ooo,1731769026392.299131947f15f9c44bcfe22cd6623096. 2024-11-16T14:57:07,667 INFO [StoreOpener-4f0d635e77a1e3dd04c4d107bb0f1564-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily21 of region 4f0d635e77a1e3dd04c4d107bb0f1564 2024-11-16T14:57:07,668 DEBUG [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(7752): Opening region: {ENCODED => 299131947f15f9c44bcfe22cd6623096, NAME => 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ooo,1731769026392.299131947f15f9c44bcfe22cd6623096.', STARTKEY => 'ooo', ENDKEY => 'ppp'} 2024-11-16T14:57:07,668 DEBUG [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true 299131947f15f9c44bcfe22cd6623096 2024-11-16T14:57:07,669 DEBUG [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(898): Instantiated testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ooo,1731769026392.299131947f15f9c44bcfe22cd6623096.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-16T14:57:07,669 DEBUG [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(7794): checking encryption for 299131947f15f9c44bcfe22cd6623096 2024-11-16T14:57:07,669 DEBUG [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(7797): checking classloading for 299131947f15f9c44bcfe22cd6623096 2024-11-16T14:57:07,670 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=43, ppid=24, state=RUNNABLE, hasLock=false; OpenRegionProcedure 7f11f79823720d664105d8643dfa862d, server=fc3fda29f660,39089,1731769023769 because future has completed 2024-11-16T14:57:07,670 INFO [StoreOpener-4f0d635e77a1e3dd04c4d107bb0f1564-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 4f0d635e77a1e3dd04c4d107bb0f1564 columnFamilyName colfamily21 2024-11-16T14:57:07,670 DEBUG [StoreOpener-4f0d635e77a1e3dd04c4d107bb0f1564-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:57:07,671 INFO [StoreOpener-4f0d635e77a1e3dd04c4d107bb0f1564-1 {}] regionserver.HStore(327): Store=4f0d635e77a1e3dd04c4d107bb0f1564/colfamily21, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-16T14:57:07,671 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=28 updating hbase:meta row=dc4fcaece350cd66f6bf88cfbdff6f15, regionState=OPEN, openSeqNum=2, regionLocation=fc3fda29f660,42985,1731769023951 2024-11-16T14:57:07,671 INFO [StoreOpener-4f0d635e77a1e3dd04c4d107bb0f1564-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily31 of region 4f0d635e77a1e3dd04c4d107bb0f1564 2024-11-16T14:57:07,671 INFO [StoreOpener-299131947f15f9c44bcfe22cd6623096-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily11 of region 299131947f15f9c44bcfe22cd6623096 2024-11-16T14:57:07,672 INFO [StoreOpener-dde7f54607f11c5fb4271e361e82dabd-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region dde7f54607f11c5fb4271e361e82dabd columnFamilyName colfamily11 2024-11-16T14:57:07,672 DEBUG [StoreOpener-dde7f54607f11c5fb4271e361e82dabd-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:57:07,674 INFO [StoreOpener-dde7f54607f11c5fb4271e361e82dabd-1 {}] regionserver.HStore(327): Store=dde7f54607f11c5fb4271e361e82dabd/colfamily11, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-16T14:57:07,674 INFO [StoreOpener-dde7f54607f11c5fb4271e361e82dabd-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily21 of region dde7f54607f11c5fb4271e361e82dabd 2024-11-16T14:57:07,674 INFO [StoreOpener-299131947f15f9c44bcfe22cd6623096-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 299131947f15f9c44bcfe22cd6623096 columnFamilyName colfamily11 2024-11-16T14:57:07,675 DEBUG [StoreOpener-299131947f15f9c44bcfe22cd6623096-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:57:07,674 INFO [StoreOpener-4f0d635e77a1e3dd04c4d107bb0f1564-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 4f0d635e77a1e3dd04c4d107bb0f1564 columnFamilyName colfamily31 2024-11-16T14:57:07,675 DEBUG [StoreOpener-4f0d635e77a1e3dd04c4d107bb0f1564-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:57:07,675 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=31, ppid=28, state=RUNNABLE, hasLock=false; OpenRegionProcedure dc4fcaece350cd66f6bf88cfbdff6f15, server=fc3fda29f660,42985,1731769023951 because future has completed 2024-11-16T14:57:07,675 INFO [StoreOpener-4f0d635e77a1e3dd04c4d107bb0f1564-1 {}] regionserver.HStore(327): Store=4f0d635e77a1e3dd04c4d107bb0f1564/colfamily31, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-16T14:57:07,676 INFO [StoreOpener-299131947f15f9c44bcfe22cd6623096-1 {}] regionserver.HStore(327): Store=299131947f15f9c44bcfe22cd6623096/colfamily11, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-16T14:57:07,676 DEBUG [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(1038): replaying wal for 4f0d635e77a1e3dd04c4d107bb0f1564 2024-11-16T14:57:07,676 INFO [StoreOpener-299131947f15f9c44bcfe22cd6623096-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily21 of region 299131947f15f9c44bcfe22cd6623096 2024-11-16T14:57:07,677 INFO [StoreOpener-dde7f54607f11c5fb4271e361e82dabd-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region dde7f54607f11c5fb4271e361e82dabd columnFamilyName colfamily21 2024-11-16T14:57:07,677 DEBUG [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42223/user/jenkins/test-data/a8176cb1-2ebb-b530-b3c7-53c9e53c186e/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/4f0d635e77a1e3dd04c4d107bb0f1564 2024-11-16T14:57:07,677 DEBUG [StoreOpener-dde7f54607f11c5fb4271e361e82dabd-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:57:07,677 DEBUG [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42223/user/jenkins/test-data/a8176cb1-2ebb-b530-b3c7-53c9e53c186e/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/4f0d635e77a1e3dd04c4d107bb0f1564 2024-11-16T14:57:07,678 INFO [StoreOpener-dde7f54607f11c5fb4271e361e82dabd-1 {}] regionserver.HStore(327): Store=dde7f54607f11c5fb4271e361e82dabd/colfamily21, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-16T14:57:07,678 INFO [StoreOpener-dde7f54607f11c5fb4271e361e82dabd-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily31 of region dde7f54607f11c5fb4271e361e82dabd 2024-11-16T14:57:07,679 INFO [StoreOpener-299131947f15f9c44bcfe22cd6623096-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 299131947f15f9c44bcfe22cd6623096 columnFamilyName colfamily21 2024-11-16T14:57:07,679 DEBUG [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(1048): stopping wal replay for 4f0d635e77a1e3dd04c4d107bb0f1564 2024-11-16T14:57:07,679 DEBUG [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(1060): Cleaning up temporary data for 4f0d635e77a1e3dd04c4d107bb0f1564 2024-11-16T14:57:07,679 DEBUG [StoreOpener-299131947f15f9c44bcfe22cd6623096-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:57:07,680 INFO [StoreOpener-299131947f15f9c44bcfe22cd6623096-1 {}] regionserver.HStore(327): Store=299131947f15f9c44bcfe22cd6623096/colfamily21, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-16T14:57:07,680 DEBUG [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-16T14:57:07,681 INFO [StoreOpener-299131947f15f9c44bcfe22cd6623096-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily31 of region 299131947f15f9c44bcfe22cd6623096 2024-11-16T14:57:07,681 INFO [StoreOpener-dde7f54607f11c5fb4271e361e82dabd-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region dde7f54607f11c5fb4271e361e82dabd columnFamilyName colfamily31 2024-11-16T14:57:07,681 DEBUG [StoreOpener-dde7f54607f11c5fb4271e361e82dabd-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:57:07,683 DEBUG [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(1093): writing seq id for 4f0d635e77a1e3dd04c4d107bb0f1564 2024-11-16T14:57:07,683 INFO [StoreOpener-299131947f15f9c44bcfe22cd6623096-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 299131947f15f9c44bcfe22cd6623096 columnFamilyName colfamily31 2024-11-16T14:57:07,683 DEBUG [StoreOpener-299131947f15f9c44bcfe22cd6623096-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:57:07,683 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=43, resume processing ppid=24 2024-11-16T14:57:07,683 INFO [StoreOpener-dde7f54607f11c5fb4271e361e82dabd-1 {}] regionserver.HStore(327): Store=dde7f54607f11c5fb4271e361e82dabd/colfamily31, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-16T14:57:07,683 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=43, ppid=24, state=SUCCESS, hasLock=false; OpenRegionProcedure 7f11f79823720d664105d8643dfa862d, server=fc3fda29f660,39089,1731769023769 in 390 msec 2024-11-16T14:57:07,683 DEBUG [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(1038): replaying wal for dde7f54607f11c5fb4271e361e82dabd 2024-11-16T14:57:07,683 INFO [StoreOpener-299131947f15f9c44bcfe22cd6623096-1 {}] regionserver.HStore(327): Store=299131947f15f9c44bcfe22cd6623096/colfamily31, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-16T14:57:07,684 DEBUG [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1038): replaying wal for 299131947f15f9c44bcfe22cd6623096 2024-11-16T14:57:07,684 DEBUG [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42223/user/jenkins/test-data/a8176cb1-2ebb-b530-b3c7-53c9e53c186e/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/dde7f54607f11c5fb4271e361e82dabd 2024-11-16T14:57:07,685 DEBUG [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42223/user/jenkins/test-data/a8176cb1-2ebb-b530-b3c7-53c9e53c186e/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/dde7f54607f11c5fb4271e361e82dabd 2024-11-16T14:57:07,685 DEBUG [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42223/user/jenkins/test-data/a8176cb1-2ebb-b530-b3c7-53c9e53c186e/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/299131947f15f9c44bcfe22cd6623096 2024-11-16T14:57:07,685 DEBUG [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42223/user/jenkins/test-data/a8176cb1-2ebb-b530-b3c7-53c9e53c186e/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/299131947f15f9c44bcfe22cd6623096 2024-11-16T14:57:07,687 DEBUG [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(1048): stopping wal replay for dde7f54607f11c5fb4271e361e82dabd 2024-11-16T14:57:07,687 DEBUG [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(1060): Cleaning up temporary data for dde7f54607f11c5fb4271e361e82dabd 2024-11-16T14:57:07,687 DEBUG [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:42223/user/jenkins/test-data/a8176cb1-2ebb-b530-b3c7-53c9e53c186e/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/4f0d635e77a1e3dd04c4d107bb0f1564/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-16T14:57:07,687 DEBUG [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1048): stopping wal replay for 299131947f15f9c44bcfe22cd6623096 2024-11-16T14:57:07,687 DEBUG [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1060): Cleaning up temporary data for 299131947f15f9c44bcfe22cd6623096 2024-11-16T14:57:07,687 DEBUG [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-16T14:57:07,688 INFO [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(1114): Opened 4f0d635e77a1e3dd04c4d107bb0f1564; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=64778915, jitterRate=-0.03471894562244415}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-16T14:57:07,688 DEBUG [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 4f0d635e77a1e3dd04c4d107bb0f1564 2024-11-16T14:57:07,688 DEBUG [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(1006): Region open journal for 4f0d635e77a1e3dd04c4d107bb0f1564: Running coprocessor pre-open hook at 1731769027656Writing region info on filesystem at 1731769027656Initializing all the Stores at 1731769027658 (+2 ms)Instantiating store for column family {NAME => 'colfamily11', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731769027658Instantiating store for column family {NAME => 'colfamily21', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731769027662 (+4 ms)Instantiating store for column family {NAME => 'colfamily31', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731769027662Cleaning up temporary data from old regions at 1731769027679 (+17 ms)Running coprocessor post-open hooks at 1731769027688 (+9 ms)Region opened successfully at 1731769027688 2024-11-16T14:57:07,688 DEBUG [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-16T14:57:07,689 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=24, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=7f11f79823720d664105d8643dfa862d, ASSIGN in 644 msec 2024-11-16T14:57:07,690 DEBUG [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(1093): writing seq id for dde7f54607f11c5fb4271e361e82dabd 2024-11-16T14:57:07,690 INFO [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegionServer(2236): Post open deploy tasks for testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,,1731769026392.4f0d635e77a1e3dd04c4d107bb0f1564., pid=56, masterSystemTime=1731769027460 2024-11-16T14:57:07,691 DEBUG [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1093): writing seq id for 299131947f15f9c44bcfe22cd6623096 2024-11-16T14:57:07,694 DEBUG [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegionServer(2266): Finished post open deploy task for testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,,1731769026392.4f0d635e77a1e3dd04c4d107bb0f1564. 2024-11-16T14:57:07,694 INFO [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] handler.AssignRegionHandler(153): Opened testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,,1731769026392.4f0d635e77a1e3dd04c4d107bb0f1564. 2024-11-16T14:57:07,694 INFO [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] handler.AssignRegionHandler(132): Open testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,eee,1731769026392.fd3944fc2b78246a853fb2232c747c35. 2024-11-16T14:57:07,695 DEBUG [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(7752): Opening region: {ENCODED => fd3944fc2b78246a853fb2232c747c35, NAME => 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,eee,1731769026392.fd3944fc2b78246a853fb2232c747c35.', STARTKEY => 'eee', ENDKEY => 'fff'} 2024-11-16T14:57:07,695 DEBUG [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true fd3944fc2b78246a853fb2232c747c35 2024-11-16T14:57:07,695 DEBUG [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(898): Instantiated testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,eee,1731769026392.fd3944fc2b78246a853fb2232c747c35.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-16T14:57:07,695 DEBUG [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(7794): checking encryption for fd3944fc2b78246a853fb2232c747c35 2024-11-16T14:57:07,695 DEBUG [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(7797): checking classloading for fd3944fc2b78246a853fb2232c747c35 2024-11-16T14:57:07,697 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=31, resume processing ppid=28 2024-11-16T14:57:07,697 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=31, ppid=28, state=SUCCESS, hasLock=false; OpenRegionProcedure dc4fcaece350cd66f6bf88cfbdff6f15, server=fc3fda29f660,42985,1731769023951 in 474 msec 2024-11-16T14:57:07,698 INFO [StoreOpener-fd3944fc2b78246a853fb2232c747c35-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily11 of region fd3944fc2b78246a853fb2232c747c35 2024-11-16T14:57:07,699 DEBUG [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:42223/user/jenkins/test-data/a8176cb1-2ebb-b530-b3c7-53c9e53c186e/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/dde7f54607f11c5fb4271e361e82dabd/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-16T14:57:07,699 DEBUG [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:42223/user/jenkins/test-data/a8176cb1-2ebb-b530-b3c7-53c9e53c186e/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/299131947f15f9c44bcfe22cd6623096/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-16T14:57:07,699 INFO [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(1114): Opened dde7f54607f11c5fb4271e361e82dabd; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=61195377, jitterRate=-0.0881178230047226}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-16T14:57:07,700 DEBUG [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(1122): Running coprocessor post-open hooks for dde7f54607f11c5fb4271e361e82dabd 2024-11-16T14:57:07,700 INFO [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1114): Opened 299131947f15f9c44bcfe22cd6623096; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=60286227, jitterRate=-0.10166521370410919}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-16T14:57:07,700 DEBUG [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 299131947f15f9c44bcfe22cd6623096 2024-11-16T14:57:07,700 DEBUG [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(1006): Region open journal for dde7f54607f11c5fb4271e361e82dabd: Running coprocessor pre-open hook at 1731769027661Writing region info on filesystem at 1731769027661Initializing all the Stores at 1731769027663 (+2 ms)Instantiating store for column family {NAME => 'colfamily11', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731769027663Instantiating store for column family {NAME => 'colfamily21', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731769027666 (+3 ms)Instantiating store for column family {NAME => 'colfamily31', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731769027666Cleaning up temporary data from old regions at 1731769027687 (+21 ms)Running coprocessor post-open hooks at 1731769027700 (+13 ms)Region opened successfully at 1731769027700 2024-11-16T14:57:07,700 DEBUG [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1006): Region open journal for 299131947f15f9c44bcfe22cd6623096: Running coprocessor pre-open hook at 1731769027669Writing region info on filesystem at 1731769027669Initializing all the Stores at 1731769027671 (+2 ms)Instantiating store for column family {NAME => 'colfamily11', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731769027671Instantiating store for column family {NAME => 'colfamily21', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731769027671Instantiating store for column family {NAME => 'colfamily31', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731769027671Cleaning up temporary data from old regions at 1731769027687 (+16 ms)Running coprocessor post-open hooks at 1731769027700 (+13 ms)Region opened successfully at 1731769027700 2024-11-16T14:57:07,701 INFO [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegionServer(2236): Post open deploy tasks for testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ooo,1731769026392.299131947f15f9c44bcfe22cd6623096., pid=32, masterSystemTime=1731769027371 2024-11-16T14:57:07,701 INFO [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegionServer(2236): Post open deploy tasks for testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ddd,1731769026392.dde7f54607f11c5fb4271e361e82dabd., pid=48, masterSystemTime=1731769027425 2024-11-16T14:57:07,702 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=28, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=dc4fcaece350cd66f6bf88cfbdff6f15, ASSIGN in 658 msec 2024-11-16T14:57:07,703 INFO [StoreOpener-fd3944fc2b78246a853fb2232c747c35-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region fd3944fc2b78246a853fb2232c747c35 columnFamilyName colfamily11 2024-11-16T14:57:07,704 DEBUG [StoreOpener-fd3944fc2b78246a853fb2232c747c35-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:57:07,704 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=4f0d635e77a1e3dd04c4d107bb0f1564, regionState=OPEN, openSeqNum=2, regionLocation=fc3fda29f660,38897,1731769023879 2024-11-16T14:57:07,704 INFO [StoreOpener-fd3944fc2b78246a853fb2232c747c35-1 {}] regionserver.HStore(327): Store=fd3944fc2b78246a853fb2232c747c35/colfamily11, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-16T14:57:07,704 INFO [StoreOpener-fd3944fc2b78246a853fb2232c747c35-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily21 of region fd3944fc2b78246a853fb2232c747c35 2024-11-16T14:57:07,706 DEBUG [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegionServer(2266): Finished post open deploy task for testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ddd,1731769026392.dde7f54607f11c5fb4271e361e82dabd. 2024-11-16T14:57:07,706 INFO [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] handler.AssignRegionHandler(153): Opened testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ddd,1731769026392.dde7f54607f11c5fb4271e361e82dabd. 2024-11-16T14:57:07,706 INFO [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] handler.AssignRegionHandler(132): Open testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,www,1731769026392.62a6d124ad9d161782c0632102c766de. 2024-11-16T14:57:07,706 DEBUG [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(7752): Opening region: {ENCODED => 62a6d124ad9d161782c0632102c766de, NAME => 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,www,1731769026392.62a6d124ad9d161782c0632102c766de.', STARTKEY => 'www', ENDKEY => 'xxx'} 2024-11-16T14:57:07,707 DEBUG [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true 62a6d124ad9d161782c0632102c766de 2024-11-16T14:57:07,707 DEBUG [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(898): Instantiated testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,www,1731769026392.62a6d124ad9d161782c0632102c766de.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-16T14:57:07,707 DEBUG [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(7794): checking encryption for 62a6d124ad9d161782c0632102c766de 2024-11-16T14:57:07,707 DEBUG [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(7797): checking classloading for 62a6d124ad9d161782c0632102c766de 2024-11-16T14:57:07,707 INFO [StoreOpener-fd3944fc2b78246a853fb2232c747c35-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region fd3944fc2b78246a853fb2232c747c35 columnFamilyName colfamily21 2024-11-16T14:57:07,707 DEBUG [StoreOpener-fd3944fc2b78246a853fb2232c747c35-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:57:07,707 DEBUG [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegionServer(2266): Finished post open deploy task for testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ooo,1731769026392.299131947f15f9c44bcfe22cd6623096. 2024-11-16T14:57:07,707 INFO [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] handler.AssignRegionHandler(153): Opened testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ooo,1731769026392.299131947f15f9c44bcfe22cd6623096. 2024-11-16T14:57:07,708 INFO [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] handler.AssignRegionHandler(132): Open testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,lll,1731769026392.266565e60d7419a2a30ceec1b0f6519b. 2024-11-16T14:57:07,708 DEBUG [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(7752): Opening region: {ENCODED => 266565e60d7419a2a30ceec1b0f6519b, NAME => 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,lll,1731769026392.266565e60d7419a2a30ceec1b0f6519b.', STARTKEY => 'lll', ENDKEY => 'mmm'} 2024-11-16T14:57:07,708 DEBUG [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true 266565e60d7419a2a30ceec1b0f6519b 2024-11-16T14:57:07,708 DEBUG [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(898): Instantiated testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,lll,1731769026392.266565e60d7419a2a30ceec1b0f6519b.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-16T14:57:07,709 DEBUG [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(7794): checking encryption for 266565e60d7419a2a30ceec1b0f6519b 2024-11-16T14:57:07,709 DEBUG [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(7797): checking classloading for 266565e60d7419a2a30ceec1b0f6519b 2024-11-16T14:57:07,709 INFO [StoreOpener-fd3944fc2b78246a853fb2232c747c35-1 {}] regionserver.HStore(327): Store=fd3944fc2b78246a853fb2232c747c35/colfamily21, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-16T14:57:07,709 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=56, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 4f0d635e77a1e3dd04c4d107bb0f1564, server=fc3fda29f660,38897,1731769023879 because future has completed 2024-11-16T14:57:07,709 INFO [StoreOpener-fd3944fc2b78246a853fb2232c747c35-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily31 of region fd3944fc2b78246a853fb2232c747c35 2024-11-16T14:57:07,710 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=19 updating hbase:meta row=299131947f15f9c44bcfe22cd6623096, regionState=OPEN, openSeqNum=2, regionLocation=fc3fda29f660,42985,1731769023951 2024-11-16T14:57:07,710 INFO [StoreOpener-62a6d124ad9d161782c0632102c766de-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily11 of region 62a6d124ad9d161782c0632102c766de 2024-11-16T14:57:07,713 INFO [StoreOpener-fd3944fc2b78246a853fb2232c747c35-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region fd3944fc2b78246a853fb2232c747c35 columnFamilyName colfamily31 2024-11-16T14:57:07,713 DEBUG [StoreOpener-fd3944fc2b78246a853fb2232c747c35-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:57:07,713 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=32, ppid=19, state=RUNNABLE, hasLock=false; OpenRegionProcedure 299131947f15f9c44bcfe22cd6623096, server=fc3fda29f660,42985,1731769023951 because future has completed 2024-11-16T14:57:07,714 INFO [StoreOpener-62a6d124ad9d161782c0632102c766de-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 62a6d124ad9d161782c0632102c766de columnFamilyName colfamily11 2024-11-16T14:57:07,714 DEBUG [StoreOpener-62a6d124ad9d161782c0632102c766de-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:57:07,714 INFO [StoreOpener-fd3944fc2b78246a853fb2232c747c35-1 {}] regionserver.HStore(327): Store=fd3944fc2b78246a853fb2232c747c35/colfamily31, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-16T14:57:07,715 DEBUG [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(1038): replaying wal for fd3944fc2b78246a853fb2232c747c35 2024-11-16T14:57:07,715 INFO [StoreOpener-62a6d124ad9d161782c0632102c766de-1 {}] regionserver.HStore(327): Store=62a6d124ad9d161782c0632102c766de/colfamily11, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-16T14:57:07,715 INFO [StoreOpener-62a6d124ad9d161782c0632102c766de-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily21 of region 62a6d124ad9d161782c0632102c766de 2024-11-16T14:57:07,716 DEBUG [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42223/user/jenkins/test-data/a8176cb1-2ebb-b530-b3c7-53c9e53c186e/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/fd3944fc2b78246a853fb2232c747c35 2024-11-16T14:57:07,716 DEBUG [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42223/user/jenkins/test-data/a8176cb1-2ebb-b530-b3c7-53c9e53c186e/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/fd3944fc2b78246a853fb2232c747c35 2024-11-16T14:57:07,717 INFO [StoreOpener-62a6d124ad9d161782c0632102c766de-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 62a6d124ad9d161782c0632102c766de columnFamilyName colfamily21 2024-11-16T14:57:07,717 DEBUG [StoreOpener-62a6d124ad9d161782c0632102c766de-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:57:07,718 INFO [StoreOpener-62a6d124ad9d161782c0632102c766de-1 {}] regionserver.HStore(327): Store=62a6d124ad9d161782c0632102c766de/colfamily21, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-16T14:57:07,718 INFO [StoreOpener-62a6d124ad9d161782c0632102c766de-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily31 of region 62a6d124ad9d161782c0632102c766de 2024-11-16T14:57:07,718 DEBUG [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(1048): stopping wal replay for fd3944fc2b78246a853fb2232c747c35 2024-11-16T14:57:07,718 DEBUG [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(1060): Cleaning up temporary data for fd3944fc2b78246a853fb2232c747c35 2024-11-16T14:57:07,719 DEBUG [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-16T14:57:07,720 INFO [StoreOpener-62a6d124ad9d161782c0632102c766de-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 62a6d124ad9d161782c0632102c766de columnFamilyName colfamily31 2024-11-16T14:57:07,720 DEBUG [StoreOpener-62a6d124ad9d161782c0632102c766de-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:57:07,720 INFO [StoreOpener-62a6d124ad9d161782c0632102c766de-1 {}] regionserver.HStore(327): Store=62a6d124ad9d161782c0632102c766de/colfamily31, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-16T14:57:07,720 DEBUG [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(1038): replaying wal for 62a6d124ad9d161782c0632102c766de 2024-11-16T14:57:07,721 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=56, resume processing ppid=5 2024-11-16T14:57:07,721 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=56, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure 4f0d635e77a1e3dd04c4d107bb0f1564, server=fc3fda29f660,38897,1731769023879 in 355 msec 2024-11-16T14:57:07,721 DEBUG [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(1093): writing seq id for fd3944fc2b78246a853fb2232c747c35 2024-11-16T14:57:07,721 DEBUG [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42223/user/jenkins/test-data/a8176cb1-2ebb-b530-b3c7-53c9e53c186e/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/62a6d124ad9d161782c0632102c766de 2024-11-16T14:57:07,722 DEBUG [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42223/user/jenkins/test-data/a8176cb1-2ebb-b530-b3c7-53c9e53c186e/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/62a6d124ad9d161782c0632102c766de 2024-11-16T14:57:07,723 DEBUG [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(1048): stopping wal replay for 62a6d124ad9d161782c0632102c766de 2024-11-16T14:57:07,724 INFO [StoreOpener-266565e60d7419a2a30ceec1b0f6519b-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily11 of region 266565e60d7419a2a30ceec1b0f6519b 2024-11-16T14:57:07,724 DEBUG [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(1060): Cleaning up temporary data for 62a6d124ad9d161782c0632102c766de 2024-11-16T14:57:07,725 DEBUG [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-16T14:57:07,725 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=4f0d635e77a1e3dd04c4d107bb0f1564, ASSIGN in 682 msec 2024-11-16T14:57:07,727 INFO [StoreOpener-266565e60d7419a2a30ceec1b0f6519b-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 266565e60d7419a2a30ceec1b0f6519b columnFamilyName colfamily11 2024-11-16T14:57:07,727 DEBUG [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(1093): writing seq id for 62a6d124ad9d161782c0632102c766de 2024-11-16T14:57:07,727 DEBUG [StoreOpener-266565e60d7419a2a30ceec1b0f6519b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:57:07,728 DEBUG [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:42223/user/jenkins/test-data/a8176cb1-2ebb-b530-b3c7-53c9e53c186e/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/fd3944fc2b78246a853fb2232c747c35/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-16T14:57:07,729 INFO [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(1114): Opened fd3944fc2b78246a853fb2232c747c35; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=65160816, jitterRate=-0.02902817726135254}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-16T14:57:07,729 DEBUG [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(1122): Running coprocessor post-open hooks for fd3944fc2b78246a853fb2232c747c35 2024-11-16T14:57:07,729 INFO [StoreOpener-266565e60d7419a2a30ceec1b0f6519b-1 {}] regionserver.HStore(327): Store=266565e60d7419a2a30ceec1b0f6519b/colfamily11, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-16T14:57:07,729 DEBUG [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(1006): Region open journal for fd3944fc2b78246a853fb2232c747c35: Running coprocessor pre-open hook at 1731769027695Writing region info on filesystem at 1731769027695Initializing all the Stores at 1731769027697 (+2 ms)Instantiating store for column family {NAME => 'colfamily11', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731769027697Instantiating store for column family {NAME => 'colfamily21', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731769027698 (+1 ms)Instantiating store for column family {NAME => 'colfamily31', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731769027698Cleaning up temporary data from old regions at 1731769027718 (+20 ms)Running coprocessor post-open hooks at 1731769027729 (+11 ms)Region opened successfully at 1731769027729 2024-11-16T14:57:07,729 INFO [StoreOpener-266565e60d7419a2a30ceec1b0f6519b-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily21 of region 266565e60d7419a2a30ceec1b0f6519b 2024-11-16T14:57:07,732 INFO [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegionServer(2236): Post open deploy tasks for testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,eee,1731769026392.fd3944fc2b78246a853fb2232c747c35., pid=51, masterSystemTime=1731769027460 2024-11-16T14:57:07,732 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=32, resume processing ppid=19 2024-11-16T14:57:07,732 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=32, ppid=19, state=SUCCESS, hasLock=false; OpenRegionProcedure 299131947f15f9c44bcfe22cd6623096, server=fc3fda29f660,42985,1731769023951 in 504 msec 2024-11-16T14:57:07,733 INFO [StoreOpener-266565e60d7419a2a30ceec1b0f6519b-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 266565e60d7419a2a30ceec1b0f6519b columnFamilyName colfamily21 2024-11-16T14:57:07,734 DEBUG [StoreOpener-266565e60d7419a2a30ceec1b0f6519b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:57:07,734 DEBUG [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:42223/user/jenkins/test-data/a8176cb1-2ebb-b530-b3c7-53c9e53c186e/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/62a6d124ad9d161782c0632102c766de/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-16T14:57:07,734 DEBUG [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegionServer(2266): Finished post open deploy task for testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,eee,1731769026392.fd3944fc2b78246a853fb2232c747c35. 2024-11-16T14:57:07,735 INFO [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] handler.AssignRegionHandler(153): Opened testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,eee,1731769026392.fd3944fc2b78246a853fb2232c747c35. 2024-11-16T14:57:07,735 INFO [StoreOpener-266565e60d7419a2a30ceec1b0f6519b-1 {}] regionserver.HStore(327): Store=266565e60d7419a2a30ceec1b0f6519b/colfamily21, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-16T14:57:07,735 INFO [StoreOpener-266565e60d7419a2a30ceec1b0f6519b-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily31 of region 266565e60d7419a2a30ceec1b0f6519b 2024-11-16T14:57:07,735 INFO [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(1114): Opened 62a6d124ad9d161782c0632102c766de; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=59185668, jitterRate=-0.11806482076644897}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-16T14:57:07,735 DEBUG [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 62a6d124ad9d161782c0632102c766de 2024-11-16T14:57:07,735 INFO [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] handler.AssignRegionHandler(132): Open testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,jjj,1731769026392.7eb7b75d28b808c5c3f2f96e407b4347. 2024-11-16T14:57:07,735 DEBUG [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(1006): Region open journal for 62a6d124ad9d161782c0632102c766de: Running coprocessor pre-open hook at 1731769027707Writing region info on filesystem at 1731769027707Initializing all the Stores at 1731769027709 (+2 ms)Instantiating store for column family {NAME => 'colfamily11', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731769027709Instantiating store for column family {NAME => 'colfamily21', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731769027709Instantiating store for column family {NAME => 'colfamily31', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731769027709Cleaning up temporary data from old regions at 1731769027724 (+15 ms)Running coprocessor post-open hooks at 1731769027735 (+11 ms)Region opened successfully at 1731769027735 2024-11-16T14:57:07,735 DEBUG [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(7752): Opening region: {ENCODED => 7eb7b75d28b808c5c3f2f96e407b4347, NAME => 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,jjj,1731769026392.7eb7b75d28b808c5c3f2f96e407b4347.', STARTKEY => 'jjj', ENDKEY => 'kkk'} 2024-11-16T14:57:07,736 DEBUG [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true 7eb7b75d28b808c5c3f2f96e407b4347 2024-11-16T14:57:07,736 DEBUG [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(898): Instantiated testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,jjj,1731769026392.7eb7b75d28b808c5c3f2f96e407b4347.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-16T14:57:07,736 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=19, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=299131947f15f9c44bcfe22cd6623096, ASSIGN in 694 msec 2024-11-16T14:57:07,736 DEBUG [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(7794): checking encryption for 7eb7b75d28b808c5c3f2f96e407b4347 2024-11-16T14:57:07,736 DEBUG [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(7797): checking classloading for 7eb7b75d28b808c5c3f2f96e407b4347 2024-11-16T14:57:07,736 INFO [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegionServer(2236): Post open deploy tasks for testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,www,1731769026392.62a6d124ad9d161782c0632102c766de., pid=40, masterSystemTime=1731769027425 2024-11-16T14:57:07,738 INFO [StoreOpener-266565e60d7419a2a30ceec1b0f6519b-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 266565e60d7419a2a30ceec1b0f6519b columnFamilyName colfamily31 2024-11-16T14:57:07,738 DEBUG [StoreOpener-266565e60d7419a2a30ceec1b0f6519b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:57:07,739 INFO [StoreOpener-266565e60d7419a2a30ceec1b0f6519b-1 {}] regionserver.HStore(327): Store=266565e60d7419a2a30ceec1b0f6519b/colfamily31, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-16T14:57:07,739 INFO [StoreOpener-7eb7b75d28b808c5c3f2f96e407b4347-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily11 of region 7eb7b75d28b808c5c3f2f96e407b4347 2024-11-16T14:57:07,739 DEBUG [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(1038): replaying wal for 266565e60d7419a2a30ceec1b0f6519b 2024-11-16T14:57:07,739 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=9 updating hbase:meta row=fd3944fc2b78246a853fb2232c747c35, regionState=OPEN, openSeqNum=2, regionLocation=fc3fda29f660,38897,1731769023879 2024-11-16T14:57:07,740 DEBUG [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegionServer(2266): Finished post open deploy task for testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,www,1731769026392.62a6d124ad9d161782c0632102c766de. 2024-11-16T14:57:07,740 INFO [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] handler.AssignRegionHandler(153): Opened testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,www,1731769026392.62a6d124ad9d161782c0632102c766de. 2024-11-16T14:57:07,740 INFO [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] handler.AssignRegionHandler(132): Open testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,kkk,1731769026392.b4f73326a2a6a279d503e557c6ed452f. 2024-11-16T14:57:07,740 DEBUG [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(7752): Opening region: {ENCODED => b4f73326a2a6a279d503e557c6ed452f, NAME => 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,kkk,1731769026392.b4f73326a2a6a279d503e557c6ed452f.', STARTKEY => 'kkk', ENDKEY => 'lll'} 2024-11-16T14:57:07,740 DEBUG [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true b4f73326a2a6a279d503e557c6ed452f 2024-11-16T14:57:07,741 DEBUG [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(898): Instantiated testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,kkk,1731769026392.b4f73326a2a6a279d503e557c6ed452f.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-16T14:57:07,741 DEBUG [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(7794): checking encryption for b4f73326a2a6a279d503e557c6ed452f 2024-11-16T14:57:07,741 DEBUG [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(7797): checking classloading for b4f73326a2a6a279d503e557c6ed452f 2024-11-16T14:57:07,741 INFO [StoreOpener-7eb7b75d28b808c5c3f2f96e407b4347-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 7eb7b75d28b808c5c3f2f96e407b4347 columnFamilyName colfamily11 2024-11-16T14:57:07,741 DEBUG [StoreOpener-7eb7b75d28b808c5c3f2f96e407b4347-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:57:07,743 DEBUG [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42223/user/jenkins/test-data/a8176cb1-2ebb-b530-b3c7-53c9e53c186e/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/266565e60d7419a2a30ceec1b0f6519b 2024-11-16T14:57:07,743 INFO [StoreOpener-7eb7b75d28b808c5c3f2f96e407b4347-1 {}] regionserver.HStore(327): Store=7eb7b75d28b808c5c3f2f96e407b4347/colfamily11, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-16T14:57:07,743 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=51, ppid=9, state=RUNNABLE, hasLock=false; OpenRegionProcedure fd3944fc2b78246a853fb2232c747c35, server=fc3fda29f660,38897,1731769023879 because future has completed 2024-11-16T14:57:07,743 INFO [StoreOpener-7eb7b75d28b808c5c3f2f96e407b4347-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily21 of region 7eb7b75d28b808c5c3f2f96e407b4347 2024-11-16T14:57:07,743 DEBUG [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42223/user/jenkins/test-data/a8176cb1-2ebb-b530-b3c7-53c9e53c186e/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/266565e60d7419a2a30ceec1b0f6519b 2024-11-16T14:57:07,744 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=27 updating hbase:meta row=62a6d124ad9d161782c0632102c766de, regionState=OPEN, openSeqNum=2, regionLocation=fc3fda29f660,39089,1731769023769 2024-11-16T14:57:07,745 DEBUG [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(1048): stopping wal replay for 266565e60d7419a2a30ceec1b0f6519b 2024-11-16T14:57:07,745 DEBUG [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(1060): Cleaning up temporary data for 266565e60d7419a2a30ceec1b0f6519b 2024-11-16T14:57:07,745 INFO [StoreOpener-7eb7b75d28b808c5c3f2f96e407b4347-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 7eb7b75d28b808c5c3f2f96e407b4347 columnFamilyName colfamily21 2024-11-16T14:57:07,745 DEBUG [StoreOpener-7eb7b75d28b808c5c3f2f96e407b4347-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:57:07,746 DEBUG [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-16T14:57:07,747 INFO [StoreOpener-7eb7b75d28b808c5c3f2f96e407b4347-1 {}] regionserver.HStore(327): Store=7eb7b75d28b808c5c3f2f96e407b4347/colfamily21, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-16T14:57:07,747 INFO [StoreOpener-7eb7b75d28b808c5c3f2f96e407b4347-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily31 of region 7eb7b75d28b808c5c3f2f96e407b4347 2024-11-16T14:57:07,748 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=40, ppid=27, state=RUNNABLE, hasLock=false; OpenRegionProcedure 62a6d124ad9d161782c0632102c766de, server=fc3fda29f660,39089,1731769023769 because future has completed 2024-11-16T14:57:07,748 DEBUG [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(1093): writing seq id for 266565e60d7419a2a30ceec1b0f6519b 2024-11-16T14:57:07,749 INFO [StoreOpener-7eb7b75d28b808c5c3f2f96e407b4347-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 7eb7b75d28b808c5c3f2f96e407b4347 columnFamilyName colfamily31 2024-11-16T14:57:07,750 DEBUG [StoreOpener-7eb7b75d28b808c5c3f2f96e407b4347-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:57:07,751 INFO [StoreOpener-7eb7b75d28b808c5c3f2f96e407b4347-1 {}] regionserver.HStore(327): Store=7eb7b75d28b808c5c3f2f96e407b4347/colfamily31, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-16T14:57:07,751 INFO [StoreOpener-b4f73326a2a6a279d503e557c6ed452f-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily11 of region b4f73326a2a6a279d503e557c6ed452f 2024-11-16T14:57:07,751 DEBUG [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(1038): replaying wal for 7eb7b75d28b808c5c3f2f96e407b4347 2024-11-16T14:57:07,752 DEBUG [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42223/user/jenkins/test-data/a8176cb1-2ebb-b530-b3c7-53c9e53c186e/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/7eb7b75d28b808c5c3f2f96e407b4347 2024-11-16T14:57:07,753 DEBUG [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42223/user/jenkins/test-data/a8176cb1-2ebb-b530-b3c7-53c9e53c186e/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/7eb7b75d28b808c5c3f2f96e407b4347 2024-11-16T14:57:07,754 INFO [StoreOpener-b4f73326a2a6a279d503e557c6ed452f-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region b4f73326a2a6a279d503e557c6ed452f columnFamilyName colfamily11 2024-11-16T14:57:07,754 DEBUG [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(1048): stopping wal replay for 7eb7b75d28b808c5c3f2f96e407b4347 2024-11-16T14:57:07,754 DEBUG [StoreOpener-b4f73326a2a6a279d503e557c6ed452f-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:57:07,754 DEBUG [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(1060): Cleaning up temporary data for 7eb7b75d28b808c5c3f2f96e407b4347 2024-11-16T14:57:07,755 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=51, resume processing ppid=9 2024-11-16T14:57:07,755 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=51, ppid=9, state=SUCCESS, hasLock=false; OpenRegionProcedure fd3944fc2b78246a853fb2232c747c35, server=fc3fda29f660,38897,1731769023879 in 414 msec 2024-11-16T14:57:07,755 DEBUG [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-16T14:57:07,756 DEBUG [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:42223/user/jenkins/test-data/a8176cb1-2ebb-b530-b3c7-53c9e53c186e/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/266565e60d7419a2a30ceec1b0f6519b/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-16T14:57:07,756 INFO [StoreOpener-b4f73326a2a6a279d503e557c6ed452f-1 {}] regionserver.HStore(327): Store=b4f73326a2a6a279d503e557c6ed452f/colfamily11, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-16T14:57:07,758 INFO [StoreOpener-b4f73326a2a6a279d503e557c6ed452f-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily21 of region b4f73326a2a6a279d503e557c6ed452f 2024-11-16T14:57:07,758 DEBUG [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(1093): writing seq id for 7eb7b75d28b808c5c3f2f96e407b4347 2024-11-16T14:57:07,759 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=43615 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=fc3fda29f660,39089,1731769023769, table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=62a6d124ad9d161782c0632102c766de. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-16T14:57:07,760 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=9, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=fd3944fc2b78246a853fb2232c747c35, ASSIGN in 716 msec 2024-11-16T14:57:07,760 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=43615 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=fc3fda29f660,42985,1731769023951, table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=35ef9a7f35bb0b08ec1d53b82d3f9777. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-16T14:57:07,761 INFO [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(1114): Opened 266565e60d7419a2a30ceec1b0f6519b; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=67297071, jitterRate=0.002804502844810486}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-16T14:57:07,761 DEBUG [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 266565e60d7419a2a30ceec1b0f6519b 2024-11-16T14:57:07,761 DEBUG [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(1006): Region open journal for 266565e60d7419a2a30ceec1b0f6519b: Running coprocessor pre-open hook at 1731769027709Writing region info on filesystem at 1731769027709Initializing all the Stores at 1731769027712 (+3 ms)Instantiating store for column family {NAME => 'colfamily11', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731769027712Instantiating store for column family {NAME => 'colfamily21', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731769027722 (+10 ms)Instantiating store for column family {NAME => 'colfamily31', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731769027722Cleaning up temporary data from old regions at 1731769027745 (+23 ms)Running coprocessor post-open hooks at 1731769027761 (+16 ms)Region opened successfully at 1731769027761 2024-11-16T14:57:07,762 INFO [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegionServer(2236): Post open deploy tasks for testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,lll,1731769026392.266565e60d7419a2a30ceec1b0f6519b., pid=36, masterSystemTime=1731769027371 2024-11-16T14:57:07,762 INFO [StoreOpener-b4f73326a2a6a279d503e557c6ed452f-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region b4f73326a2a6a279d503e557c6ed452f columnFamilyName colfamily21 2024-11-16T14:57:07,763 DEBUG [StoreOpener-b4f73326a2a6a279d503e557c6ed452f-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:57:07,763 DEBUG [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:42223/user/jenkins/test-data/a8176cb1-2ebb-b530-b3c7-53c9e53c186e/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/7eb7b75d28b808c5c3f2f96e407b4347/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-16T14:57:07,764 INFO [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(1114): Opened 7eb7b75d28b808c5c3f2f96e407b4347; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=64667961, jitterRate=-0.036372289061546326}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-16T14:57:07,764 DEBUG [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 7eb7b75d28b808c5c3f2f96e407b4347 2024-11-16T14:57:07,764 DEBUG [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(1006): Region open journal for 7eb7b75d28b808c5c3f2f96e407b4347: Running coprocessor pre-open hook at 1731769027736Writing region info on filesystem at 1731769027736Initializing all the Stores at 1731769027738 (+2 ms)Instantiating store for column family {NAME => 'colfamily11', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731769027738Instantiating store for column family {NAME => 'colfamily21', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731769027738Instantiating store for column family {NAME => 'colfamily31', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731769027738Cleaning up temporary data from old regions at 1731769027754 (+16 ms)Running coprocessor post-open hooks at 1731769027764 (+10 ms)Region opened successfully at 1731769027764 2024-11-16T14:57:07,764 INFO [StoreOpener-b4f73326a2a6a279d503e557c6ed452f-1 {}] regionserver.HStore(327): Store=b4f73326a2a6a279d503e557c6ed452f/colfamily21, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-16T14:57:07,765 INFO [StoreOpener-b4f73326a2a6a279d503e557c6ed452f-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily31 of region b4f73326a2a6a279d503e557c6ed452f 2024-11-16T14:57:07,767 INFO [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegionServer(2236): Post open deploy tasks for testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,jjj,1731769026392.7eb7b75d28b808c5c3f2f96e407b4347., pid=50, masterSystemTime=1731769027460 2024-11-16T14:57:07,767 DEBUG [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegionServer(2266): Finished post open deploy task for testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,lll,1731769026392.266565e60d7419a2a30ceec1b0f6519b. 2024-11-16T14:57:07,768 INFO [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] handler.AssignRegionHandler(153): Opened testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,lll,1731769026392.266565e60d7419a2a30ceec1b0f6519b. 2024-11-16T14:57:07,769 INFO [StoreOpener-b4f73326a2a6a279d503e557c6ed452f-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region b4f73326a2a6a279d503e557c6ed452f columnFamilyName colfamily31 2024-11-16T14:57:07,769 DEBUG [StoreOpener-b4f73326a2a6a279d503e557c6ed452f-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:57:07,770 INFO [StoreOpener-b4f73326a2a6a279d503e557c6ed452f-1 {}] regionserver.HStore(327): Store=b4f73326a2a6a279d503e557c6ed452f/colfamily31, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-16T14:57:07,771 DEBUG [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(1038): replaying wal for b4f73326a2a6a279d503e557c6ed452f 2024-11-16T14:57:07,771 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=40, resume processing ppid=27 2024-11-16T14:57:07,771 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=40, ppid=27, state=SUCCESS, hasLock=false; OpenRegionProcedure 62a6d124ad9d161782c0632102c766de, server=fc3fda29f660,39089,1731769023769 in 491 msec 2024-11-16T14:57:07,772 DEBUG [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegionServer(2266): Finished post open deploy task for testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,jjj,1731769026392.7eb7b75d28b808c5c3f2f96e407b4347. 2024-11-16T14:57:07,772 INFO [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] handler.AssignRegionHandler(153): Opened testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,jjj,1731769026392.7eb7b75d28b808c5c3f2f96e407b4347. 2024-11-16T14:57:07,772 INFO [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] handler.AssignRegionHandler(132): Open testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,yyy,1731769026392.426ff231e487ce9f8492776ca1dacc85. 2024-11-16T14:57:07,772 DEBUG [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42223/user/jenkins/test-data/a8176cb1-2ebb-b530-b3c7-53c9e53c186e/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/b4f73326a2a6a279d503e557c6ed452f 2024-11-16T14:57:07,772 DEBUG [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(7752): Opening region: {ENCODED => 426ff231e487ce9f8492776ca1dacc85, NAME => 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,yyy,1731769026392.426ff231e487ce9f8492776ca1dacc85.', STARTKEY => 'yyy', ENDKEY => 'zzz'} 2024-11-16T14:57:07,773 DEBUG [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42223/user/jenkins/test-data/a8176cb1-2ebb-b530-b3c7-53c9e53c186e/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/b4f73326a2a6a279d503e557c6ed452f 2024-11-16T14:57:07,773 DEBUG [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true 426ff231e487ce9f8492776ca1dacc85 2024-11-16T14:57:07,773 DEBUG [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(898): Instantiated testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,yyy,1731769026392.426ff231e487ce9f8492776ca1dacc85.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-16T14:57:07,773 DEBUG [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(7794): checking encryption for 426ff231e487ce9f8492776ca1dacc85 2024-11-16T14:57:07,773 DEBUG [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(7797): checking classloading for 426ff231e487ce9f8492776ca1dacc85 2024-11-16T14:57:07,774 DEBUG [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(1048): stopping wal replay for b4f73326a2a6a279d503e557c6ed452f 2024-11-16T14:57:07,775 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=27, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=62a6d124ad9d161782c0632102c766de, ASSIGN in 732 msec 2024-11-16T14:57:07,775 DEBUG [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(1060): Cleaning up temporary data for b4f73326a2a6a279d503e557c6ed452f 2024-11-16T14:57:07,775 INFO [StoreOpener-426ff231e487ce9f8492776ca1dacc85-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily11 of region 426ff231e487ce9f8492776ca1dacc85 2024-11-16T14:57:07,775 DEBUG [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-16T14:57:07,776 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=14 updating hbase:meta row=7eb7b75d28b808c5c3f2f96e407b4347, regionState=OPEN, openSeqNum=2, regionLocation=fc3fda29f660,38897,1731769023879 2024-11-16T14:57:07,778 DEBUG [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(1093): writing seq id for b4f73326a2a6a279d503e557c6ed452f 2024-11-16T14:57:07,778 INFO [StoreOpener-426ff231e487ce9f8492776ca1dacc85-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 426ff231e487ce9f8492776ca1dacc85 columnFamilyName colfamily11 2024-11-16T14:57:07,778 DEBUG [StoreOpener-426ff231e487ce9f8492776ca1dacc85-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:57:07,779 INFO [StoreOpener-426ff231e487ce9f8492776ca1dacc85-1 {}] regionserver.HStore(327): Store=426ff231e487ce9f8492776ca1dacc85/colfamily11, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-16T14:57:07,779 INFO [StoreOpener-426ff231e487ce9f8492776ca1dacc85-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily21 of region 426ff231e487ce9f8492776ca1dacc85 2024-11-16T14:57:07,780 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=50, ppid=14, state=RUNNABLE, hasLock=false; OpenRegionProcedure 7eb7b75d28b808c5c3f2f96e407b4347, server=fc3fda29f660,38897,1731769023879 because future has completed 2024-11-16T14:57:07,780 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=16 updating hbase:meta row=266565e60d7419a2a30ceec1b0f6519b, regionState=OPEN, openSeqNum=2, regionLocation=fc3fda29f660,42985,1731769023951 2024-11-16T14:57:07,782 INFO [StoreOpener-426ff231e487ce9f8492776ca1dacc85-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 426ff231e487ce9f8492776ca1dacc85 columnFamilyName colfamily21 2024-11-16T14:57:07,782 DEBUG [StoreOpener-426ff231e487ce9f8492776ca1dacc85-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:57:07,783 INFO [StoreOpener-426ff231e487ce9f8492776ca1dacc85-1 {}] regionserver.HStore(327): Store=426ff231e487ce9f8492776ca1dacc85/colfamily21, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-16T14:57:07,783 INFO [StoreOpener-426ff231e487ce9f8492776ca1dacc85-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily31 of region 426ff231e487ce9f8492776ca1dacc85 2024-11-16T14:57:07,784 DEBUG [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:42223/user/jenkins/test-data/a8176cb1-2ebb-b530-b3c7-53c9e53c186e/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/b4f73326a2a6a279d503e557c6ed452f/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-16T14:57:07,784 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=36, ppid=16, state=RUNNABLE, hasLock=false; OpenRegionProcedure 266565e60d7419a2a30ceec1b0f6519b, server=fc3fda29f660,42985,1731769023951 because future has completed 2024-11-16T14:57:07,785 INFO [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(1114): Opened b4f73326a2a6a279d503e557c6ed452f; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=65661960, jitterRate=-0.02156054973602295}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-16T14:57:07,785 DEBUG [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(1122): Running coprocessor post-open hooks for b4f73326a2a6a279d503e557c6ed452f 2024-11-16T14:57:07,785 DEBUG [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(1006): Region open journal for b4f73326a2a6a279d503e557c6ed452f: Running coprocessor pre-open hook at 1731769027741Writing region info on filesystem at 1731769027741Initializing all the Stores at 1731769027744 (+3 ms)Instantiating store for column family {NAME => 'colfamily11', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731769027744Instantiating store for column family {NAME => 'colfamily21', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731769027751 (+7 ms)Instantiating store for column family {NAME => 'colfamily31', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731769027751Cleaning up temporary data from old regions at 1731769027775 (+24 ms)Running coprocessor post-open hooks at 1731769027785 (+10 ms)Region opened successfully at 1731769027785 2024-11-16T14:57:07,785 INFO [StoreOpener-426ff231e487ce9f8492776ca1dacc85-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 426ff231e487ce9f8492776ca1dacc85 columnFamilyName colfamily31 2024-11-16T14:57:07,785 DEBUG [StoreOpener-426ff231e487ce9f8492776ca1dacc85-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:57:07,786 INFO [StoreOpener-426ff231e487ce9f8492776ca1dacc85-1 {}] regionserver.HStore(327): Store=426ff231e487ce9f8492776ca1dacc85/colfamily31, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-16T14:57:07,786 DEBUG [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(1038): replaying wal for 426ff231e487ce9f8492776ca1dacc85 2024-11-16T14:57:07,786 INFO [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegionServer(2236): Post open deploy tasks for testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,kkk,1731769026392.b4f73326a2a6a279d503e557c6ed452f., pid=45, masterSystemTime=1731769027425 2024-11-16T14:57:07,787 DEBUG [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42223/user/jenkins/test-data/a8176cb1-2ebb-b530-b3c7-53c9e53c186e/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/426ff231e487ce9f8492776ca1dacc85 2024-11-16T14:57:07,788 DEBUG [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42223/user/jenkins/test-data/a8176cb1-2ebb-b530-b3c7-53c9e53c186e/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/426ff231e487ce9f8492776ca1dacc85 2024-11-16T14:57:07,790 DEBUG [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegionServer(2266): Finished post open deploy task for testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,kkk,1731769026392.b4f73326a2a6a279d503e557c6ed452f. 2024-11-16T14:57:07,790 INFO [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] handler.AssignRegionHandler(153): Opened testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,kkk,1731769026392.b4f73326a2a6a279d503e557c6ed452f. 2024-11-16T14:57:07,791 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=50, resume processing ppid=14 2024-11-16T14:57:07,791 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=50, ppid=14, state=SUCCESS, hasLock=false; OpenRegionProcedure 7eb7b75d28b808c5c3f2f96e407b4347, server=fc3fda29f660,38897,1731769023879 in 454 msec 2024-11-16T14:57:07,791 DEBUG [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(1048): stopping wal replay for 426ff231e487ce9f8492776ca1dacc85 2024-11-16T14:57:07,791 DEBUG [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(1060): Cleaning up temporary data for 426ff231e487ce9f8492776ca1dacc85 2024-11-16T14:57:07,792 DEBUG [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-16T14:57:07,793 DEBUG [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(1093): writing seq id for 426ff231e487ce9f8492776ca1dacc85 2024-11-16T14:57:07,794 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=14, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=7eb7b75d28b808c5c3f2f96e407b4347, ASSIGN in 752 msec 2024-11-16T14:57:07,796 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=15 updating hbase:meta row=b4f73326a2a6a279d503e557c6ed452f, regionState=OPEN, openSeqNum=2, regionLocation=fc3fda29f660,39089,1731769023769 2024-11-16T14:57:07,797 DEBUG [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:42223/user/jenkins/test-data/a8176cb1-2ebb-b530-b3c7-53c9e53c186e/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/426ff231e487ce9f8492776ca1dacc85/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-16T14:57:07,798 INFO [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(1114): Opened 426ff231e487ce9f8492776ca1dacc85; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=65131174, jitterRate=-0.02946987748146057}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-16T14:57:07,798 DEBUG [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 426ff231e487ce9f8492776ca1dacc85 2024-11-16T14:57:07,799 DEBUG [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(1006): Region open journal for 426ff231e487ce9f8492776ca1dacc85: Running coprocessor pre-open hook at 1731769027773Writing region info on filesystem at 1731769027773Initializing all the Stores at 1731769027774 (+1 ms)Instantiating store for column family {NAME => 'colfamily11', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731769027774Instantiating store for column family {NAME => 'colfamily21', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731769027775 (+1 ms)Instantiating store for column family {NAME => 'colfamily31', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731769027775Cleaning up temporary data from old regions at 1731769027791 (+16 ms)Running coprocessor post-open hooks at 1731769027798 (+7 ms)Region opened successfully at 1731769027798 2024-11-16T14:57:07,800 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=45, ppid=15, state=RUNNABLE, hasLock=false; OpenRegionProcedure b4f73326a2a6a279d503e557c6ed452f, server=fc3fda29f660,39089,1731769023769 because future has completed 2024-11-16T14:57:07,800 INFO [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegionServer(2236): Post open deploy tasks for testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,yyy,1731769026392.426ff231e487ce9f8492776ca1dacc85., pid=46, masterSystemTime=1731769027460 2024-11-16T14:57:07,803 DEBUG [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegionServer(2266): Finished post open deploy task for testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,yyy,1731769026392.426ff231e487ce9f8492776ca1dacc85. 2024-11-16T14:57:07,804 INFO [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] handler.AssignRegionHandler(153): Opened testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,yyy,1731769026392.426ff231e487ce9f8492776ca1dacc85. 2024-11-16T14:57:07,804 INFO [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] handler.AssignRegionHandler(132): Open testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,vvv,1731769026392.b5566c7e10a0e001c7f58316bdd43c2f. 2024-11-16T14:57:07,804 DEBUG [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(7752): Opening region: {ENCODED => b5566c7e10a0e001c7f58316bdd43c2f, NAME => 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,vvv,1731769026392.b5566c7e10a0e001c7f58316bdd43c2f.', STARTKEY => 'vvv', ENDKEY => 'www'} 2024-11-16T14:57:07,804 DEBUG [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true b5566c7e10a0e001c7f58316bdd43c2f 2024-11-16T14:57:07,805 DEBUG [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(898): Instantiated testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,vvv,1731769026392.b5566c7e10a0e001c7f58316bdd43c2f.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-16T14:57:07,805 DEBUG [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(7794): checking encryption for b5566c7e10a0e001c7f58316bdd43c2f 2024-11-16T14:57:07,805 DEBUG [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(7797): checking classloading for b5566c7e10a0e001c7f58316bdd43c2f 2024-11-16T14:57:07,805 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=36, resume processing ppid=16 2024-11-16T14:57:07,805 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=36, ppid=16, state=SUCCESS, hasLock=false; OpenRegionProcedure 266565e60d7419a2a30ceec1b0f6519b, server=fc3fda29f660,42985,1731769023951 in 553 msec 2024-11-16T14:57:07,807 INFO [StoreOpener-b5566c7e10a0e001c7f58316bdd43c2f-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily11 of region b5566c7e10a0e001c7f58316bdd43c2f 2024-11-16T14:57:07,808 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=16, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=266565e60d7419a2a30ceec1b0f6519b, ASSIGN in 767 msec 2024-11-16T14:57:07,809 INFO [StoreOpener-b5566c7e10a0e001c7f58316bdd43c2f-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region b5566c7e10a0e001c7f58316bdd43c2f columnFamilyName colfamily11 2024-11-16T14:57:07,809 DEBUG [StoreOpener-b5566c7e10a0e001c7f58316bdd43c2f-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:57:07,810 INFO [StoreOpener-b5566c7e10a0e001c7f58316bdd43c2f-1 {}] regionserver.HStore(327): Store=b5566c7e10a0e001c7f58316bdd43c2f/colfamily11, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-16T14:57:07,810 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=29 updating hbase:meta row=426ff231e487ce9f8492776ca1dacc85, regionState=OPEN, openSeqNum=2, regionLocation=fc3fda29f660,38897,1731769023879 2024-11-16T14:57:07,810 INFO [StoreOpener-b5566c7e10a0e001c7f58316bdd43c2f-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily21 of region b5566c7e10a0e001c7f58316bdd43c2f 2024-11-16T14:57:07,812 INFO [StoreOpener-b5566c7e10a0e001c7f58316bdd43c2f-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region b5566c7e10a0e001c7f58316bdd43c2f columnFamilyName colfamily21 2024-11-16T14:57:07,812 DEBUG [StoreOpener-b5566c7e10a0e001c7f58316bdd43c2f-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:57:07,813 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=46, ppid=29, state=RUNNABLE, hasLock=false; OpenRegionProcedure 426ff231e487ce9f8492776ca1dacc85, server=fc3fda29f660,38897,1731769023879 because future has completed 2024-11-16T14:57:07,813 INFO [StoreOpener-b5566c7e10a0e001c7f58316bdd43c2f-1 {}] regionserver.HStore(327): Store=b5566c7e10a0e001c7f58316bdd43c2f/colfamily21, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-16T14:57:07,814 INFO [StoreOpener-b5566c7e10a0e001c7f58316bdd43c2f-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily31 of region b5566c7e10a0e001c7f58316bdd43c2f 2024-11-16T14:57:07,816 INFO [StoreOpener-b5566c7e10a0e001c7f58316bdd43c2f-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region b5566c7e10a0e001c7f58316bdd43c2f columnFamilyName colfamily31 2024-11-16T14:57:07,816 DEBUG [StoreOpener-b5566c7e10a0e001c7f58316bdd43c2f-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T14:57:07,817 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=45, resume processing ppid=15 2024-11-16T14:57:07,817 INFO [StoreOpener-b5566c7e10a0e001c7f58316bdd43c2f-1 {}] regionserver.HStore(327): Store=b5566c7e10a0e001c7f58316bdd43c2f/colfamily31, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-16T14:57:07,817 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=45, ppid=15, state=SUCCESS, hasLock=false; OpenRegionProcedure b4f73326a2a6a279d503e557c6ed452f, server=fc3fda29f660,39089,1731769023769 in 514 msec 2024-11-16T14:57:07,817 DEBUG [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(1038): replaying wal for b5566c7e10a0e001c7f58316bdd43c2f 2024-11-16T14:57:07,819 DEBUG [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42223/user/jenkins/test-data/a8176cb1-2ebb-b530-b3c7-53c9e53c186e/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/b5566c7e10a0e001c7f58316bdd43c2f 2024-11-16T14:57:07,819 DEBUG [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42223/user/jenkins/test-data/a8176cb1-2ebb-b530-b3c7-53c9e53c186e/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/b5566c7e10a0e001c7f58316bdd43c2f 2024-11-16T14:57:07,821 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=15, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=b4f73326a2a6a279d503e557c6ed452f, ASSIGN in 779 msec 2024-11-16T14:57:07,821 DEBUG [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(1048): stopping wal replay for b5566c7e10a0e001c7f58316bdd43c2f 2024-11-16T14:57:07,821 DEBUG [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(1060): Cleaning up temporary data for b5566c7e10a0e001c7f58316bdd43c2f 2024-11-16T14:57:07,822 DEBUG [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-16T14:57:07,824 DEBUG [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(1093): writing seq id for b5566c7e10a0e001c7f58316bdd43c2f 2024-11-16T14:57:07,827 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=46, resume processing ppid=29 2024-11-16T14:57:07,827 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=46, ppid=29, state=SUCCESS, hasLock=false; OpenRegionProcedure 426ff231e487ce9f8492776ca1dacc85, server=fc3fda29f660,38897,1731769023879 in 516 msec 2024-11-16T14:57:07,827 DEBUG [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:42223/user/jenkins/test-data/a8176cb1-2ebb-b530-b3c7-53c9e53c186e/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/b5566c7e10a0e001c7f58316bdd43c2f/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-16T14:57:07,828 INFO [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(1114): Opened b5566c7e10a0e001c7f58316bdd43c2f; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=63481210, jitterRate=-0.054056257009506226}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-16T14:57:07,828 DEBUG [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(1122): Running coprocessor post-open hooks for b5566c7e10a0e001c7f58316bdd43c2f 2024-11-16T14:57:07,828 DEBUG [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(1006): Region open journal for b5566c7e10a0e001c7f58316bdd43c2f: Running coprocessor pre-open hook at 1731769027805Writing region info on filesystem at 1731769027805Initializing all the Stores at 1731769027806 (+1 ms)Instantiating store for column family {NAME => 'colfamily11', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731769027806Instantiating store for column family {NAME => 'colfamily21', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731769027807 (+1 ms)Instantiating store for column family {NAME => 'colfamily31', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731769027807Cleaning up temporary data from old regions at 1731769027821 (+14 ms)Running coprocessor post-open hooks at 1731769027828 (+7 ms)Region opened successfully at 1731769027828 2024-11-16T14:57:07,829 INFO [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegionServer(2236): Post open deploy tasks for testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,vvv,1731769026392.b5566c7e10a0e001c7f58316bdd43c2f., pid=49, masterSystemTime=1731769027460 2024-11-16T14:57:07,830 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=29, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=426ff231e487ce9f8492776ca1dacc85, ASSIGN in 788 msec 2024-11-16T14:57:07,832 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=8 updating hbase:meta row=dde7f54607f11c5fb4271e361e82dabd, regionState=OPEN, openSeqNum=2, regionLocation=fc3fda29f660,39089,1731769023769 2024-11-16T14:57:07,832 DEBUG [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegionServer(2266): Finished post open deploy task for testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,vvv,1731769026392.b5566c7e10a0e001c7f58316bdd43c2f. 2024-11-16T14:57:07,832 INFO [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] handler.AssignRegionHandler(153): Opened testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,vvv,1731769026392.b5566c7e10a0e001c7f58316bdd43c2f. 2024-11-16T14:57:07,835 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=48, ppid=8, state=RUNNABLE, hasLock=false; OpenRegionProcedure dde7f54607f11c5fb4271e361e82dabd, server=fc3fda29f660,39089,1731769023769 because future has completed 2024-11-16T14:57:07,835 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=26 updating hbase:meta row=b5566c7e10a0e001c7f58316bdd43c2f, regionState=OPEN, openSeqNum=2, regionLocation=fc3fda29f660,38897,1731769023879 2024-11-16T14:57:07,838 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=49, ppid=26, state=RUNNABLE, hasLock=false; OpenRegionProcedure b5566c7e10a0e001c7f58316bdd43c2f, server=fc3fda29f660,38897,1731769023879 because future has completed 2024-11-16T14:57:07,842 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=48, resume processing ppid=8 2024-11-16T14:57:07,842 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=48, ppid=8, state=SUCCESS, hasLock=false; OpenRegionProcedure dde7f54607f11c5fb4271e361e82dabd, server=fc3fda29f660,39089,1731769023769 in 523 msec 2024-11-16T14:57:07,844 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=8, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=dde7f54607f11c5fb4271e361e82dabd, ASSIGN in 803 msec 2024-11-16T14:57:07,847 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=49, resume processing ppid=26 2024-11-16T14:57:07,847 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=49, ppid=26, state=SUCCESS, hasLock=false; OpenRegionProcedure b5566c7e10a0e001c7f58316bdd43c2f, server=fc3fda29f660,38897,1731769023879 in 519 msec 2024-11-16T14:57:07,849 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=26, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=b5566c7e10a0e001c7f58316bdd43c2f, ASSIGN in 808 msec 2024-11-16T14:57:07,850 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=20 updating hbase:meta row=7f5c9df3a544933227f58b052f743f72, regionState=OPEN, openSeqNum=2, regionLocation=fc3fda29f660,38897,1731769023879 2024-11-16T14:57:07,853 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=54, ppid=20, state=RUNNABLE, hasLock=false; OpenRegionProcedure 7f5c9df3a544933227f58b052f743f72, server=fc3fda29f660,38897,1731769023879 because future has completed 2024-11-16T14:57:07,856 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=38, resume processing ppid=25 2024-11-16T14:57:07,857 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=38, ppid=25, state=SUCCESS, hasLock=false; OpenRegionProcedure 35ef9a7f35bb0b08ec1d53b82d3f9777, server=fc3fda29f660,42985,1731769023951 in 597 msec 2024-11-16T14:57:07,859 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=25, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=35ef9a7f35bb0b08ec1d53b82d3f9777, ASSIGN in 818 msec 2024-11-16T14:57:07,865 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=54, resume processing ppid=20 2024-11-16T14:57:07,865 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=54, ppid=20, state=SUCCESS, hasLock=false; OpenRegionProcedure 7f5c9df3a544933227f58b052f743f72, server=fc3fda29f660,38897,1731769023879 in 512 msec 2024-11-16T14:57:07,868 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=20, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=7f5c9df3a544933227f58b052f743f72, ASSIGN in 826 msec 2024-11-16T14:57:07,869 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=23 updating hbase:meta row=33054df03c1be51dc860405880be6075, regionState=OPEN, openSeqNum=2, regionLocation=fc3fda29f660,38897,1731769023879 2024-11-16T14:57:07,872 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=55, ppid=23, state=RUNNABLE, hasLock=false; OpenRegionProcedure 33054df03c1be51dc860405880be6075, server=fc3fda29f660,38897,1731769023879 because future has completed 2024-11-16T14:57:07,875 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=43615 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=fc3fda29f660,38897,1731769023879, table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=33054df03c1be51dc860405880be6075. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-16T14:57:07,879 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=55, resume processing ppid=23 2024-11-16T14:57:07,880 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=55, ppid=23, state=SUCCESS, hasLock=false; OpenRegionProcedure 33054df03c1be51dc860405880be6075, server=fc3fda29f660,38897,1731769023879 in 524 msec 2024-11-16T14:57:07,884 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=23, resume processing ppid=4 2024-11-16T14:57:07,884 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=23, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=33054df03c1be51dc860405880be6075, ASSIGN in 841 msec 2024-11-16T14:57:07,886 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-16T14:57:07,886 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731769027886"}]},"ts":"1731769027886"} 2024-11-16T14:57:07,889 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, state=ENABLED in hbase:meta 2024-11-16T14:57:07,890 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true execute state=CREATE_TABLE_POST_OPERATION 2024-11-16T14:57:07,896 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true in 1.4840 sec 2024-11-16T14:57:08,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43615 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-16T14:57:08,582 INFO [RPCClient-NioEventLoopGroup-6-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true completed 2024-11-16T14:57:08,582 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true get assigned. Timeout = 60000ms 2024-11-16T14:57:08,584 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-16T14:57:08,600 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38897 {}] regionserver.StoreScanner(1138): Switch to stream read (scanned=32802 bytes) of info 2024-11-16T14:57:08,610 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true assigned to meta. Checking AM states. 2024-11-16T14:57:08,611 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-16T14:57:08,612 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true assigned. 2024-11-16T14:57:08,668 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='aab', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,,1731769026392.4f0d635e77a1e3dd04c4d107bb0f1564., hostname=fc3fda29f660,38897,1731769023879, seqNum=2] 2024-11-16T14:57:08,671 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='aaa', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,,1731769026392.4f0d635e77a1e3dd04c4d107bb0f1564., hostname=fc3fda29f660,38897,1731769023879, seqNum=2] 2024-11-16T14:57:08,673 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='aac', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,,1731769026392.4f0d635e77a1e3dd04c4d107bb0f1564., hostname=fc3fda29f660,38897,1731769023879, seqNum=2] 2024-11-16T14:57:08,674 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='aad', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,,1731769026392.4f0d635e77a1e3dd04c4d107bb0f1564., hostname=fc3fda29f660,38897,1731769023879, seqNum=2] 2024-11-16T14:57:08,676 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='aae', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,,1731769026392.4f0d635e77a1e3dd04c4d107bb0f1564., hostname=fc3fda29f660,38897,1731769023879, seqNum=2] 2024-11-16T14:57:08,677 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='aaf', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,,1731769026392.4f0d635e77a1e3dd04c4d107bb0f1564., hostname=fc3fda29f660,38897,1731769023879, seqNum=2] 2024-11-16T14:57:08,678 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='aag', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,,1731769026392.4f0d635e77a1e3dd04c4d107bb0f1564., hostname=fc3fda29f660,38897,1731769023879, seqNum=2] 2024-11-16T14:57:08,736 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='bbb', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,bbb,1731769026392.c3dde5f5f668d914608a2987d4f4c001., hostname=fc3fda29f660,42985,1731769023951, seqNum=2] 2024-11-16T14:57:08,738 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='bbc', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,bbb,1731769026392.c3dde5f5f668d914608a2987d4f4c001., hostname=fc3fda29f660,42985,1731769023951, seqNum=2] 2024-11-16T14:57:08,740 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='bbd', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,bbb,1731769026392.c3dde5f5f668d914608a2987d4f4c001., hostname=fc3fda29f660,42985,1731769023951, seqNum=2] 2024-11-16T14:57:08,745 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='bbe', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,bbb,1731769026392.c3dde5f5f668d914608a2987d4f4c001., hostname=fc3fda29f660,42985,1731769023951, seqNum=2] 2024-11-16T14:57:08,785 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='ccc', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ccc,1731769026392.5bb22144956dcd43c9b1d957232b11e7., hostname=fc3fda29f660,39089,1731769023769, seqNum=2] 2024-11-16T14:57:08,787 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='ccd', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ccc,1731769026392.5bb22144956dcd43c9b1d957232b11e7., hostname=fc3fda29f660,39089,1731769023769, seqNum=2] 2024-11-16T14:57:08,789 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='cce', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ccc,1731769026392.5bb22144956dcd43c9b1d957232b11e7., hostname=fc3fda29f660,39089,1731769023769, seqNum=2] 2024-11-16T14:57:08,791 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='ccf', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ccc,1731769026392.5bb22144956dcd43c9b1d957232b11e7., hostname=fc3fda29f660,39089,1731769023769, seqNum=2] 2024-11-16T14:57:08,795 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='cch', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ccc,1731769026392.5bb22144956dcd43c9b1d957232b11e7., hostname=fc3fda29f660,39089,1731769023769, seqNum=2] 2024-11-16T14:57:08,797 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='ccg', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ccc,1731769026392.5bb22144956dcd43c9b1d957232b11e7., hostname=fc3fda29f660,39089,1731769023769, seqNum=2] 2024-11-16T14:57:08,798 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='cci', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ccc,1731769026392.5bb22144956dcd43c9b1d957232b11e7., hostname=fc3fda29f660,39089,1731769023769, seqNum=2] 2024-11-16T14:57:08,859 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='ddd', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ddd,1731769026392.dde7f54607f11c5fb4271e361e82dabd., hostname=fc3fda29f660,39089,1731769023769, seqNum=2] 2024-11-16T14:57:08,862 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='dde', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ddd,1731769026392.dde7f54607f11c5fb4271e361e82dabd., hostname=fc3fda29f660,39089,1731769023769, seqNum=2] 2024-11-16T14:57:08,863 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='ddf', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ddd,1731769026392.dde7f54607f11c5fb4271e361e82dabd., hostname=fc3fda29f660,39089,1731769023769, seqNum=2] 2024-11-16T14:57:08,865 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='ddg', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ddd,1731769026392.dde7f54607f11c5fb4271e361e82dabd., hostname=fc3fda29f660,39089,1731769023769, seqNum=2] 2024-11-16T14:57:08,867 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='ddh', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ddd,1731769026392.dde7f54607f11c5fb4271e361e82dabd., hostname=fc3fda29f660,39089,1731769023769, seqNum=2] 2024-11-16T14:57:08,869 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='ddi', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ddd,1731769026392.dde7f54607f11c5fb4271e361e82dabd., hostname=fc3fda29f660,39089,1731769023769, seqNum=2] 2024-11-16T14:57:08,920 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='eee', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,eee,1731769026392.fd3944fc2b78246a853fb2232c747c35., hostname=fc3fda29f660,38897,1731769023879, seqNum=2] 2024-11-16T14:57:08,922 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='eef', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,eee,1731769026392.fd3944fc2b78246a853fb2232c747c35., hostname=fc3fda29f660,38897,1731769023879, seqNum=2] 2024-11-16T14:57:08,924 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='eeg', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,eee,1731769026392.fd3944fc2b78246a853fb2232c747c35., hostname=fc3fda29f660,38897,1731769023879, seqNum=2] 2024-11-16T14:57:08,926 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='eeh', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,eee,1731769026392.fd3944fc2b78246a853fb2232c747c35., hostname=fc3fda29f660,38897,1731769023879, seqNum=2] 2024-11-16T14:57:08,928 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='eei', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,eee,1731769026392.fd3944fc2b78246a853fb2232c747c35., hostname=fc3fda29f660,38897,1731769023879, seqNum=2] 2024-11-16T14:57:08,964 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='fff', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,fff,1731769026392.a5db9431e22152bd2a82e84504d2879a., hostname=fc3fda29f660,42985,1731769023951, seqNum=2] 2024-11-16T14:57:08,967 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='ffg', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,fff,1731769026392.a5db9431e22152bd2a82e84504d2879a., hostname=fc3fda29f660,42985,1731769023951, seqNum=2] 2024-11-16T14:57:08,968 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='ffh', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,fff,1731769026392.a5db9431e22152bd2a82e84504d2879a., hostname=fc3fda29f660,42985,1731769023951, seqNum=2] 2024-11-16T14:57:08,970 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='ffi', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,fff,1731769026392.a5db9431e22152bd2a82e84504d2879a., hostname=fc3fda29f660,42985,1731769023951, seqNum=2] 2024-11-16T14:57:08,972 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='ffj', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,fff,1731769026392.a5db9431e22152bd2a82e84504d2879a., hostname=fc3fda29f660,42985,1731769023951, seqNum=2] 2024-11-16T14:57:09,011 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='ggg', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ggg,1731769026392.dcd6414ddebad66e87ab37a6dba924f3., hostname=fc3fda29f660,42985,1731769023951, seqNum=2] 2024-11-16T14:57:09,013 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='ggh', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ggg,1731769026392.dcd6414ddebad66e87ab37a6dba924f3., hostname=fc3fda29f660,42985,1731769023951, seqNum=2] 2024-11-16T14:57:09,017 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='ggi', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ggg,1731769026392.dcd6414ddebad66e87ab37a6dba924f3., hostname=fc3fda29f660,42985,1731769023951, seqNum=2] 2024-11-16T14:57:09,019 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='ggj', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ggg,1731769026392.dcd6414ddebad66e87ab37a6dba924f3., hostname=fc3fda29f660,42985,1731769023951, seqNum=2] 2024-11-16T14:57:09,021 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='ggk', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ggg,1731769026392.dcd6414ddebad66e87ab37a6dba924f3., hostname=fc3fda29f660,42985,1731769023951, seqNum=2] 2024-11-16T14:57:09,023 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='ggl', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ggg,1731769026392.dcd6414ddebad66e87ab37a6dba924f3., hostname=fc3fda29f660,42985,1731769023951, seqNum=2] 2024-11-16T14:57:09,059 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='hhh', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,hhh,1731769026392.1d97df67e5110b0fe1a1e4b7c1341cbb., hostname=fc3fda29f660,42985,1731769023951, seqNum=2] 2024-11-16T14:57:09,062 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='hhi', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,hhh,1731769026392.1d97df67e5110b0fe1a1e4b7c1341cbb., hostname=fc3fda29f660,42985,1731769023951, seqNum=2] 2024-11-16T14:57:09,063 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='hhj', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,hhh,1731769026392.1d97df67e5110b0fe1a1e4b7c1341cbb., hostname=fc3fda29f660,42985,1731769023951, seqNum=2] 2024-11-16T14:57:09,066 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='hhk', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,hhh,1731769026392.1d97df67e5110b0fe1a1e4b7c1341cbb., hostname=fc3fda29f660,42985,1731769023951, seqNum=2] 2024-11-16T14:57:09,068 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='hhl', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,hhh,1731769026392.1d97df67e5110b0fe1a1e4b7c1341cbb., hostname=fc3fda29f660,42985,1731769023951, seqNum=2] 2024-11-16T14:57:09,070 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='hhm', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,hhh,1731769026392.1d97df67e5110b0fe1a1e4b7c1341cbb., hostname=fc3fda29f660,42985,1731769023951, seqNum=2] 2024-11-16T14:57:09,072 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='hhn', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,hhh,1731769026392.1d97df67e5110b0fe1a1e4b7c1341cbb., hostname=fc3fda29f660,42985,1731769023951, seqNum=2] 2024-11-16T14:57:09,074 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='hho', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,hhh,1731769026392.1d97df67e5110b0fe1a1e4b7c1341cbb., hostname=fc3fda29f660,42985,1731769023951, seqNum=2] 2024-11-16T14:57:09,109 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='iii', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,iii,1731769026392.7352dcdbf9811dcebc51a714a1de4f55., hostname=fc3fda29f660,38897,1731769023879, seqNum=2] 2024-11-16T14:57:09,111 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='iij', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,iii,1731769026392.7352dcdbf9811dcebc51a714a1de4f55., hostname=fc3fda29f660,38897,1731769023879, seqNum=2] 2024-11-16T14:57:09,113 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='iil', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,iii,1731769026392.7352dcdbf9811dcebc51a714a1de4f55., hostname=fc3fda29f660,38897,1731769023879, seqNum=2] 2024-11-16T14:57:09,115 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='iik', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,iii,1731769026392.7352dcdbf9811dcebc51a714a1de4f55., hostname=fc3fda29f660,38897,1731769023879, seqNum=2] 2024-11-16T14:57:09,118 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='iim', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,iii,1731769026392.7352dcdbf9811dcebc51a714a1de4f55., hostname=fc3fda29f660,38897,1731769023879, seqNum=2] 2024-11-16T14:57:09,119 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='iin', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,iii,1731769026392.7352dcdbf9811dcebc51a714a1de4f55., hostname=fc3fda29f660,38897,1731769023879, seqNum=2] 2024-11-16T14:57:09,152 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='jjk', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,jjj,1731769026392.7eb7b75d28b808c5c3f2f96e407b4347., hostname=fc3fda29f660,38897,1731769023879, seqNum=2] 2024-11-16T14:57:09,155 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='jjj', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,jjj,1731769026392.7eb7b75d28b808c5c3f2f96e407b4347., hostname=fc3fda29f660,38897,1731769023879, seqNum=2] 2024-11-16T14:57:09,157 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='jjm', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,jjj,1731769026392.7eb7b75d28b808c5c3f2f96e407b4347., hostname=fc3fda29f660,38897,1731769023879, seqNum=2] 2024-11-16T14:57:09,159 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='jjl', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,jjj,1731769026392.7eb7b75d28b808c5c3f2f96e407b4347., hostname=fc3fda29f660,38897,1731769023879, seqNum=2] 2024-11-16T14:57:09,161 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='jjn', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,jjj,1731769026392.7eb7b75d28b808c5c3f2f96e407b4347., hostname=fc3fda29f660,38897,1731769023879, seqNum=2] 2024-11-16T14:57:09,163 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='jjo', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,jjj,1731769026392.7eb7b75d28b808c5c3f2f96e407b4347., hostname=fc3fda29f660,38897,1731769023879, seqNum=2] 2024-11-16T14:57:09,165 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='jjp', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,jjj,1731769026392.7eb7b75d28b808c5c3f2f96e407b4347., hostname=fc3fda29f660,38897,1731769023879, seqNum=2] 2024-11-16T14:57:09,197 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='kkk', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,kkk,1731769026392.b4f73326a2a6a279d503e557c6ed452f., hostname=fc3fda29f660,39089,1731769023769, seqNum=2] 2024-11-16T14:57:09,201 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='kkl', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,kkk,1731769026392.b4f73326a2a6a279d503e557c6ed452f., hostname=fc3fda29f660,39089,1731769023769, seqNum=2] 2024-11-16T14:57:09,204 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='kkn', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,kkk,1731769026392.b4f73326a2a6a279d503e557c6ed452f., hostname=fc3fda29f660,39089,1731769023769, seqNum=2] 2024-11-16T14:57:09,206 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='kkm', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,kkk,1731769026392.b4f73326a2a6a279d503e557c6ed452f., hostname=fc3fda29f660,39089,1731769023769, seqNum=2] 2024-11-16T14:57:09,209 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='kko', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,kkk,1731769026392.b4f73326a2a6a279d503e557c6ed452f., hostname=fc3fda29f660,39089,1731769023769, seqNum=2] 2024-11-16T14:57:09,212 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='kkp', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,kkk,1731769026392.b4f73326a2a6a279d503e557c6ed452f., hostname=fc3fda29f660,39089,1731769023769, seqNum=2] 2024-11-16T14:57:09,214 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='kkq', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,kkk,1731769026392.b4f73326a2a6a279d503e557c6ed452f., hostname=fc3fda29f660,39089,1731769023769, seqNum=2] 2024-11-16T14:57:09,268 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='lll', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,lll,1731769026392.266565e60d7419a2a30ceec1b0f6519b., hostname=fc3fda29f660,42985,1731769023951, seqNum=2] 2024-11-16T14:57:09,271 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='llm', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,lll,1731769026392.266565e60d7419a2a30ceec1b0f6519b., hostname=fc3fda29f660,42985,1731769023951, seqNum=2] 2024-11-16T14:57:09,273 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='lln', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,lll,1731769026392.266565e60d7419a2a30ceec1b0f6519b., hostname=fc3fda29f660,42985,1731769023951, seqNum=2] 2024-11-16T14:57:09,275 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='llo', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,lll,1731769026392.266565e60d7419a2a30ceec1b0f6519b., hostname=fc3fda29f660,42985,1731769023951, seqNum=2] 2024-11-16T14:57:09,278 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='llp', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,lll,1731769026392.266565e60d7419a2a30ceec1b0f6519b., hostname=fc3fda29f660,42985,1731769023951, seqNum=2] 2024-11-16T14:57:09,327 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='mmm', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,mmm,1731769026392.c2acbc05ae7bb1607a5719b701b40383., hostname=fc3fda29f660,38897,1731769023879, seqNum=2] 2024-11-16T14:57:09,329 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='mmn', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,mmm,1731769026392.c2acbc05ae7bb1607a5719b701b40383., hostname=fc3fda29f660,38897,1731769023879, seqNum=2] 2024-11-16T14:57:09,331 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='mmo', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,mmm,1731769026392.c2acbc05ae7bb1607a5719b701b40383., hostname=fc3fda29f660,38897,1731769023879, seqNum=2] 2024-11-16T14:57:09,333 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='mmp', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,mmm,1731769026392.c2acbc05ae7bb1607a5719b701b40383., hostname=fc3fda29f660,38897,1731769023879, seqNum=2] 2024-11-16T14:57:09,336 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='mmq', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,mmm,1731769026392.c2acbc05ae7bb1607a5719b701b40383., hostname=fc3fda29f660,38897,1731769023879, seqNum=2] 2024-11-16T14:57:09,338 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='mmr', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,mmm,1731769026392.c2acbc05ae7bb1607a5719b701b40383., hostname=fc3fda29f660,38897,1731769023879, seqNum=2] 2024-11-16T14:57:09,369 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='nnn', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,nnn,1731769026392.994ca1526cf6c4838e65f85444853f1a., hostname=fc3fda29f660,39089,1731769023769, seqNum=2] 2024-11-16T14:57:09,371 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='nno', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,nnn,1731769026392.994ca1526cf6c4838e65f85444853f1a., hostname=fc3fda29f660,39089,1731769023769, seqNum=2] 2024-11-16T14:57:09,373 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='nnp', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,nnn,1731769026392.994ca1526cf6c4838e65f85444853f1a., hostname=fc3fda29f660,39089,1731769023769, seqNum=2] 2024-11-16T14:57:09,375 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='nnq', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,nnn,1731769026392.994ca1526cf6c4838e65f85444853f1a., hostname=fc3fda29f660,39089,1731769023769, seqNum=2] 2024-11-16T14:57:09,378 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='nnr', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,nnn,1731769026392.994ca1526cf6c4838e65f85444853f1a., hostname=fc3fda29f660,39089,1731769023769, seqNum=2] 2024-11-16T14:57:09,411 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='oop', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ooo,1731769026392.299131947f15f9c44bcfe22cd6623096., hostname=fc3fda29f660,42985,1731769023951, seqNum=2] 2024-11-16T14:57:09,413 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='ooo', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ooo,1731769026392.299131947f15f9c44bcfe22cd6623096., hostname=fc3fda29f660,42985,1731769023951, seqNum=2] 2024-11-16T14:57:09,415 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='ooq', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ooo,1731769026392.299131947f15f9c44bcfe22cd6623096., hostname=fc3fda29f660,42985,1731769023951, seqNum=2] 2024-11-16T14:57:09,418 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='oor', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ooo,1731769026392.299131947f15f9c44bcfe22cd6623096., hostname=fc3fda29f660,42985,1731769023951, seqNum=2] 2024-11-16T14:57:09,420 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='oos', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ooo,1731769026392.299131947f15f9c44bcfe22cd6623096., hostname=fc3fda29f660,42985,1731769023951, seqNum=2] 2024-11-16T14:57:09,422 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='oot', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ooo,1731769026392.299131947f15f9c44bcfe22cd6623096., hostname=fc3fda29f660,42985,1731769023951, seqNum=2] 2024-11-16T14:57:09,455 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='ppp', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ppp,1731769026392.7f5c9df3a544933227f58b052f743f72., hostname=fc3fda29f660,38897,1731769023879, seqNum=2] 2024-11-16T14:57:09,458 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='ppq', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ppp,1731769026392.7f5c9df3a544933227f58b052f743f72., hostname=fc3fda29f660,38897,1731769023879, seqNum=2] 2024-11-16T14:57:09,460 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='ppr', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ppp,1731769026392.7f5c9df3a544933227f58b052f743f72., hostname=fc3fda29f660,38897,1731769023879, seqNum=2] 2024-11-16T14:57:09,461 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='pps', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ppp,1731769026392.7f5c9df3a544933227f58b052f743f72., hostname=fc3fda29f660,38897,1731769023879, seqNum=2] 2024-11-16T14:57:09,464 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='ppt', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ppp,1731769026392.7f5c9df3a544933227f58b052f743f72., hostname=fc3fda29f660,38897,1731769023879, seqNum=2] 2024-11-16T14:57:09,498 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='qqr', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,qqq,1731769026392.18f03e2509a3a89e4a69ba87fad73940., hostname=fc3fda29f660,39089,1731769023769, seqNum=2] 2024-11-16T14:57:09,499 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='qqq', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,qqq,1731769026392.18f03e2509a3a89e4a69ba87fad73940., hostname=fc3fda29f660,39089,1731769023769, seqNum=2] 2024-11-16T14:57:09,501 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='qqs', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,qqq,1731769026392.18f03e2509a3a89e4a69ba87fad73940., hostname=fc3fda29f660,39089,1731769023769, seqNum=2] 2024-11-16T14:57:09,503 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='qqt', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,qqq,1731769026392.18f03e2509a3a89e4a69ba87fad73940., hostname=fc3fda29f660,39089,1731769023769, seqNum=2] 2024-11-16T14:57:09,505 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='qqu', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,qqq,1731769026392.18f03e2509a3a89e4a69ba87fad73940., hostname=fc3fda29f660,39089,1731769023769, seqNum=2] 2024-11-16T14:57:09,506 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='qqv', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,qqq,1731769026392.18f03e2509a3a89e4a69ba87fad73940., hostname=fc3fda29f660,39089,1731769023769, seqNum=2] 2024-11-16T14:57:09,565 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='rrr', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,rrr,1731769026392.b364a006589fe0d2474ffa45692a6efc., hostname=fc3fda29f660,42985,1731769023951, seqNum=2] 2024-11-16T14:57:09,568 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='rrs', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,rrr,1731769026392.b364a006589fe0d2474ffa45692a6efc., hostname=fc3fda29f660,42985,1731769023951, seqNum=2] 2024-11-16T14:57:09,569 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='rrt', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,rrr,1731769026392.b364a006589fe0d2474ffa45692a6efc., hostname=fc3fda29f660,42985,1731769023951, seqNum=2] 2024-11-16T14:57:09,571 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='rru', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,rrr,1731769026392.b364a006589fe0d2474ffa45692a6efc., hostname=fc3fda29f660,42985,1731769023951, seqNum=2] 2024-11-16T14:57:09,573 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='rrv', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,rrr,1731769026392.b364a006589fe0d2474ffa45692a6efc., hostname=fc3fda29f660,42985,1731769023951, seqNum=2] 2024-11-16T14:57:09,606 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='sss', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,sss,1731769026392.33054df03c1be51dc860405880be6075., hostname=fc3fda29f660,38897,1731769023879, seqNum=2] 2024-11-16T14:57:09,608 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='sst', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,sss,1731769026392.33054df03c1be51dc860405880be6075., hostname=fc3fda29f660,38897,1731769023879, seqNum=2] 2024-11-16T14:57:09,609 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='ssu', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,sss,1731769026392.33054df03c1be51dc860405880be6075., hostname=fc3fda29f660,38897,1731769023879, seqNum=2] 2024-11-16T14:57:09,611 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='ssv', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,sss,1731769026392.33054df03c1be51dc860405880be6075., hostname=fc3fda29f660,38897,1731769023879, seqNum=2] 2024-11-16T14:57:09,646 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='ttt', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ttt,1731769026392.7f11f79823720d664105d8643dfa862d., hostname=fc3fda29f660,39089,1731769023769, seqNum=2] 2024-11-16T14:57:09,648 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='ttu', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ttt,1731769026392.7f11f79823720d664105d8643dfa862d., hostname=fc3fda29f660,39089,1731769023769, seqNum=2] 2024-11-16T14:57:09,650 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='ttv', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ttt,1731769026392.7f11f79823720d664105d8643dfa862d., hostname=fc3fda29f660,39089,1731769023769, seqNum=2] 2024-11-16T14:57:09,652 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='ttw', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ttt,1731769026392.7f11f79823720d664105d8643dfa862d., hostname=fc3fda29f660,39089,1731769023769, seqNum=2] 2024-11-16T14:57:09,654 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='ttx', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ttt,1731769026392.7f11f79823720d664105d8643dfa862d., hostname=fc3fda29f660,39089,1731769023769, seqNum=2] 2024-11-16T14:57:09,687 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='uuu', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,uuu,1731769026392.35ef9a7f35bb0b08ec1d53b82d3f9777., hostname=fc3fda29f660,42985,1731769023951, seqNum=2] 2024-11-16T14:57:09,690 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='uuv', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,uuu,1731769026392.35ef9a7f35bb0b08ec1d53b82d3f9777., hostname=fc3fda29f660,42985,1731769023951, seqNum=2] 2024-11-16T14:57:09,692 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='uuw', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,uuu,1731769026392.35ef9a7f35bb0b08ec1d53b82d3f9777., hostname=fc3fda29f660,42985,1731769023951, seqNum=2] 2024-11-16T14:57:09,693 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='uux', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,uuu,1731769026392.35ef9a7f35bb0b08ec1d53b82d3f9777., hostname=fc3fda29f660,42985,1731769023951, seqNum=2] 2024-11-16T14:57:09,695 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='uuy', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,uuu,1731769026392.35ef9a7f35bb0b08ec1d53b82d3f9777., hostname=fc3fda29f660,42985,1731769023951, seqNum=2] 2024-11-16T14:57:09,727 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='vvw', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,vvv,1731769026392.b5566c7e10a0e001c7f58316bdd43c2f., hostname=fc3fda29f660,38897,1731769023879, seqNum=2] 2024-11-16T14:57:09,729 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='vvv', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,vvv,1731769026392.b5566c7e10a0e001c7f58316bdd43c2f., hostname=fc3fda29f660,38897,1731769023879, seqNum=2] 2024-11-16T14:57:09,731 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='vvx', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,vvv,1731769026392.b5566c7e10a0e001c7f58316bdd43c2f., hostname=fc3fda29f660,38897,1731769023879, seqNum=2] 2024-11-16T14:57:09,733 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='vvy', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,vvv,1731769026392.b5566c7e10a0e001c7f58316bdd43c2f., hostname=fc3fda29f660,38897,1731769023879, seqNum=2] 2024-11-16T14:57:09,735 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='vvz', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,vvv,1731769026392.b5566c7e10a0e001c7f58316bdd43c2f., hostname=fc3fda29f660,38897,1731769023879, seqNum=2] 2024-11-16T14:57:09,766 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='www', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,www,1731769026392.62a6d124ad9d161782c0632102c766de., hostname=fc3fda29f660,39089,1731769023769, seqNum=2] 2024-11-16T14:57:09,768 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='wwx', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,www,1731769026392.62a6d124ad9d161782c0632102c766de., hostname=fc3fda29f660,39089,1731769023769, seqNum=2] 2024-11-16T14:57:09,770 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='wwy', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,www,1731769026392.62a6d124ad9d161782c0632102c766de., hostname=fc3fda29f660,39089,1731769023769, seqNum=2] 2024-11-16T14:57:09,771 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='wwz', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,www,1731769026392.62a6d124ad9d161782c0632102c766de., hostname=fc3fda29f660,39089,1731769023769, seqNum=2] 2024-11-16T14:57:09,772 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='wxa', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,www,1731769026392.62a6d124ad9d161782c0632102c766de., hostname=fc3fda29f660,39089,1731769023769, seqNum=2] 2024-11-16T14:57:09,815 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='xxx', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,xxx,1731769026392.dc4fcaece350cd66f6bf88cfbdff6f15., hostname=fc3fda29f660,42985,1731769023951, seqNum=2] 2024-11-16T14:57:09,817 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='xxy', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,xxx,1731769026392.dc4fcaece350cd66f6bf88cfbdff6f15., hostname=fc3fda29f660,42985,1731769023951, seqNum=2] 2024-11-16T14:57:09,819 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='xxz', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,xxx,1731769026392.dc4fcaece350cd66f6bf88cfbdff6f15., hostname=fc3fda29f660,42985,1731769023951, seqNum=2] 2024-11-16T14:57:09,820 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='xya', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,xxx,1731769026392.dc4fcaece350cd66f6bf88cfbdff6f15., hostname=fc3fda29f660,42985,1731769023951, seqNum=2] 2024-11-16T14:57:09,821 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='xyb', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,xxx,1731769026392.dc4fcaece350cd66f6bf88cfbdff6f15., hostname=fc3fda29f660,42985,1731769023951, seqNum=2] 2024-11-16T14:57:09,866 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='yyy', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,yyy,1731769026392.426ff231e487ce9f8492776ca1dacc85., hostname=fc3fda29f660,38897,1731769023879, seqNum=2] 2024-11-16T14:57:09,868 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='yyz', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,yyy,1731769026392.426ff231e487ce9f8492776ca1dacc85., hostname=fc3fda29f660,38897,1731769023879, seqNum=2] 2024-11-16T14:57:09,870 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='yza', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,yyy,1731769026392.426ff231e487ce9f8492776ca1dacc85., hostname=fc3fda29f660,38897,1731769023879, seqNum=2] 2024-11-16T14:57:09,872 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='yzc', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,yyy,1731769026392.426ff231e487ce9f8492776ca1dacc85., hostname=fc3fda29f660,38897,1731769023879, seqNum=2] 2024-11-16T14:57:09,874 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='yzb', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,yyy,1731769026392.426ff231e487ce9f8492776ca1dacc85., hostname=fc3fda29f660,38897,1731769023879, seqNum=2] 2024-11-16T14:57:09,876 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='yzd', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,yyy,1731769026392.426ff231e487ce9f8492776ca1dacc85., hostname=fc3fda29f660,38897,1731769023879, seqNum=2] 2024-11-16T14:57:09,911 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='zzz', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,zzz,1731769026392.2f41a8c996224c818e917389bd3f5141., hostname=fc3fda29f660,39089,1731769023769, seqNum=2] 2024-11-16T14:57:09,938 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-16T14:57:09,950 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-16T14:57:09,955 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35028, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-16T14:57:09,965 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54476, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-16T14:57:09,978 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42985 {}] regionserver.RSRpcServices(2610): Large batch operation detected (greater than 5000) (HBASE-18023). Requested Number of Rows: 6327 Client: jenkins//172.17.0.2 first region in multi=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,bbb,1731769026392.c3dde5f5f668d914608a2987d4f4c001. 2024-11-16T14:57:10,260 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38897 {}] regionserver.RSRpcServices(2610): Large batch operation detected (greater than 5000) (HBASE-18023). Requested Number of Rows: 6327 Client: jenkins//172.17.0.2 first region in multi=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,,1731769026392.4f0d635e77a1e3dd04c4d107bb0f1564. 2024-11-16T14:57:10,808 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,, stopping at row=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true ,, for max=2147483647 with caching=100 2024-11-16T14:57:10,811 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38897 {}] regionserver.StoreScanner(1138): Switch to stream read (scanned=32802 bytes) of info 2024-11-16T14:57:11,067 WARN [Time-limited test {}] procedure2.ProcedureTestingUtility(193): Set Kill before store update to: true 2024-11-16T14:57:11,068 INFO [Time-limited test {}] assignment.AssignmentTestingUtil(82): KILL REGION SERVER fc3fda29f660,38897,1731769023879 2024-11-16T14:57:11,068 INFO [Time-limited test {}] hbase.SingleProcessHBaseCluster(267): Killing fc3fda29f660,38897,1731769023879 2024-11-16T14:57:11,068 ERROR [Time-limited test {}] regionserver.HRegionServer(2442): ***** ABORTING region server fc3fda29f660,38897,1731769023879: Simulated kill ***** 2024-11-16T14:57:11,069 ERROR [Time-limited test {}] regionserver.HRegionServer(2447): RegionServer abort: loaded coprocessors are: [org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint] 2024-11-16T14:57:11,071 DEBUG [Time-limited test {}] util.JSONBean(135): Listing beans for java.lang:type=Memory 2024-11-16T14:57:11,073 DEBUG [Time-limited test {}] util.JSONBean(135): Listing beans for Hadoop:service=HBase,name=RegionServer,sub=IPC 2024-11-16T14:57:11,074 DEBUG [Time-limited test {}] util.JSONBean(135): Listing beans for Hadoop:service=HBase,name=RegionServer,sub=Replication 2024-11-16T14:57:11,074 DEBUG [Time-limited test {}] util.JSONBean(135): Listing beans for Hadoop:service=HBase,name=RegionServer,sub=Server 2024-11-16T14:57:11,074 INFO [Time-limited test {}] regionserver.HRegionServer(2451): Dump of metrics as JSON on abort: { "beans": [ { "name": "java.lang:type=Memory", "modelerType": "sun.management.MemoryImpl", "ObjectPendingFinalizationCount": 0, "HeapMemoryUsage": { "committed": 1048576000, "init": 1048576000, "max": 2306867200, "used": 619276880 }, "NonHeapMemoryUsage": { "committed": 158007296, "init": 7667712, "max": -1, "used": 154845744 }, "Verbose": false, "ObjectName": "java.lang:type=Memory" } ], "beans": [], "beans": [], "beans": [] } 2024-11-16T14:57:11,076 WARN [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43615 {}] master.MasterRpcServices(700): fc3fda29f660,38897,1731769023879 reported a fatal error: ***** ABORTING region server fc3fda29f660,38897,1731769023879: Simulated kill ***** 2024-11-16T14:57:11,078 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server 'fc3fda29f660,38897,1731769023879' ***** 2024-11-16T14:57:11,078 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Simulated kill 2024-11-16T14:57:11,079 INFO [RS:1;fc3fda29f660:38897 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-16T14:57:11,079 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-16T14:57:11,079 INFO [RS:1;fc3fda29f660:38897 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager abruptly. 2024-11-16T14:57:11,079 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-16T14:57:11,079 INFO [RS:1;fc3fda29f660:38897 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager abruptly. 2024-11-16T14:57:11,079 INFO [RS:1;fc3fda29f660:38897 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-16T14:57:11,079 INFO [RS:1;fc3fda29f660:38897 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:1;fc3fda29f660:38897. 2024-11-16T14:57:11,080 DEBUG [RS:1;fc3fda29f660:38897 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-16T14:57:11,081 INFO [regionserver/fc3fda29f660:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-16T14:57:11,084 DEBUG [RS:1;fc3fda29f660:38897 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T14:57:11,090 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40641 is added to blk_1073741836_1012 (size=54214) 2024-11-16T14:57:11,090 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41671 is added to blk_1073741836_1012 (size=54214) 2024-11-16T14:57:11,091 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36531 is added to blk_1073741836_1012 (size=54214) 2024-11-16T14:57:11,097 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36531 is added to blk_1073741833_1009 (size=334915) 2024-11-16T14:57:11,097 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41671 is added to blk_1073741833_1009 (size=334915) 2024-11-16T14:57:11,097 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40641 is added to blk_1073741833_1009 (size=334915) 2024-11-16T14:57:11,098 DEBUG [RS:1;fc3fda29f660:38897 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T14:57:11,098 INFO [RS:1;fc3fda29f660:38897 {}] regionserver.LeaseManager(133): Closed leases 2024-11-16T14:57:11,098 INFO [RS:1;fc3fda29f660:38897 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:38897 2024-11-16T14:57:11,194 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43615-0x101456fc9c70000, quorum=127.0.0.1:63224, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-16T14:57:11,194 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38897-0x101456fc9c70002, quorum=127.0.0.1:63224, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/fc3fda29f660,38897,1731769023879 2024-11-16T14:57:11,194 INFO [RS:1;fc3fda29f660:38897 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-16T14:57:11,275 INFO [regionserver/fc3fda29f660:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: CompactionChecker was stopped 2024-11-16T14:57:11,275 INFO [regionserver/fc3fda29f660:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: MemstoreFlusherChore was stopped 2024-11-16T14:57:11,286 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [fc3fda29f660,38897,1731769023879] 2024-11-16T14:57:11,319 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/fc3fda29f660,38897,1731769023879 already deleted, retry=false 2024-11-16T14:57:11,319 INFO [RegionServerTracker-0 {}] master.ServerManager(695): Processing expiration of fc3fda29f660,38897,1731769023879 on fc3fda29f660,43615,1731769023046 2024-11-16T14:57:11,330 DEBUG [RegionServerTracker-0 {}] procedure2.ProcedureExecutor(1139): Stored pid=57, state=RUNNABLE:SERVER_CRASH_START, hasLock=false; ServerCrashProcedure fc3fda29f660,38897,1731769023879, splitWal=true, meta=true 2024-11-16T14:57:11,334 INFO [RegionServerTracker-0 {}] assignment.AssignmentManager(1999): Scheduled ServerCrashProcedure pid=57 for fc3fda29f660,38897,1731769023879 (carryingMeta=true) fc3fda29f660,38897,1731769023879/CRASHED/regionCount=10/lock=java.util.concurrent.locks.ReentrantReadWriteLock@6dd1ebac[Write locks = 1, Read locks = 0], oldState=ONLINE. 2024-11-16T14:57:11,334 INFO [PEWorker-1 {}] procedure.ServerCrashProcedure(169): Start pid=57, state=RUNNABLE:SERVER_CRASH_START, hasLock=true; ServerCrashProcedure fc3fda29f660,38897,1731769023879, splitWal=true, meta=true 2024-11-16T14:57:11,335 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(132): Toggle KILL before store update to: false 2024-11-16T14:57:11,335 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update: pid=57, state=RUNNABLE:SERVER_CRASH_SPLIT_META_LOGS, hasLock=true; ServerCrashProcedure fc3fda29f660,38897,1731769023879, splitWal=true, meta=true 2024-11-16T14:57:11,335 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-16T14:57:11,335 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update: pid=57, state=RUNNABLE:SERVER_CRASH_SPLIT_META_LOGS, hasLock=true; ServerCrashProcedure fc3fda29f660,38897,1731769023879, splitWal=true, meta=true at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1879) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:57:11,380 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [30,000] milli-secs(wait.for.ratio=[1]) 2024-11-16T14:57:11,382 INFO [Time-limited test {}] procedure.MasterProcedureTestingUtility(473): Restart 0 exec state=pid=57, state=RUNNABLE:SERVER_CRASH_SPLIT_META_LOGS, hasLock=true; ServerCrashProcedure fc3fda29f660,38897,1731769023879, splitWal=true, meta=true 2024-11-16T14:57:11,383 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-16T14:57:11,383 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-16T14:57:11,383 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-16T14:57:11,384 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-16T14:57:11,384 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-16T14:57:11,385 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-16T14:57:11,385 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-16T14:57:11,385 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-16T14:57:11,386 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-16T14:57:11,386 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38897-0x101456fc9c70002, quorum=127.0.0.1:63224, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-16T14:57:11,387 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38897-0x101456fc9c70002, quorum=127.0.0.1:63224, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-16T14:57:11,387 INFO [RS:1;fc3fda29f660:38897 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-16T14:57:11,387 INFO [RS:1;fc3fda29f660:38897 {}] regionserver.HRegionServer(1031): Exiting; stopping=fc3fda29f660,38897,1731769023879; zookeeper connection closed. 2024-11-16T14:57:11,388 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@4f64e367 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@4f64e367 2024-11-16T14:57:11,393 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=57, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-16T14:57:11,394 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-16T14:57:11,395 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true 2024-11-16T14:57:11,395 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=57, state=RUNNABLE:SERVER_CRASH_START, hasLock=false; ServerCrashProcedure fc3fda29f660,38897,1731769023879, splitWal=true, meta=true 2024-11-16T14:57:11,396 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=57, state=RUNNABLE:SERVER_CRASH_START, hasLock=false; ServerCrashProcedure fc3fda29f660,38897,1731769023879, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-16T14:57:11,396 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 9 msec 2024-11-16T14:57:11,398 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=fc3fda29f660,38897,1731769023879, regionLocation=fc3fda29f660,38897,1731769023879, openSeqNum=2 2024-11-16T14:57:11,398 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=fc3fda29f660,38897,1731769023879, table=hbase:meta, region=1588230740 2024-11-16T14:57:11,398 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as fc3fda29f660,38897,1731769023879, state=OPEN 2024-11-16T14:57:11,410 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39089-0x101456fc9c70001, quorum=127.0.0.1:63224, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-16T14:57:11,410 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43615-0x101456fc9c70000, quorum=127.0.0.1:63224, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-16T14:57:11,410 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42985-0x101456fc9c70003, quorum=127.0.0.1:63224, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-16T14:57:11,411 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-16T14:57:11,411 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-16T14:57:11,411 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-16T14:57:11,413 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731769061413 2024-11-16T14:57:11,413 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-16T14:57:11,413 INFO [PEWorker-1 {}] procedure.ServerCrashProcedure(169): Start pid=57, state=RUNNABLE:SERVER_CRASH_START, hasLock=true; ServerCrashProcedure fc3fda29f660,38897,1731769023879, splitWal=true, meta=true 2024-11-16T14:57:11,413 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(132): Toggle KILL before store update to: true 2024-11-16T14:57:11,415 WARN [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.NettyRpcConnection$2(409): Exception encountered while connecting to the server fc3fda29f660:38897 org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: fc3fda29f660/172.17.0.2:38897 Caused by: java.net.ConnectException: finishConnect(..) failed: Connection refused at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.newConnectException0(Errors.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.handleConnectErrno(Errors.java:131) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Socket.finishConnect(Socket.java:359) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.doFinishConnect(AbstractEpollChannel.java:715) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:692) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:57:11,416 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=fc3fda29f660,38897,1731769023879, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=fc3fda29f660,38897,1731769023879, seqNum=-1, error=java.net.ConnectException: Call to address=fc3fda29f660:38897 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: fc3fda29f660/172.17.0.2:38897 2024-11-16T14:57:11,417 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:meta,,1.1588230740, hostname=fc3fda29f660,38897,1731769023879, seqNum=-1 is java.net.ConnectException: finishConnect(..) failed: Connection refused 2024-11-16T14:57:11,417 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:meta,,1.1588230740, hostname=fc3fda29f660,38897,1731769023879, seqNum=-1 from cache 2024-11-16T14:57:11,418 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.FailedServers(52): Added failed server with address fc3fda29f660:38897 to list caused by org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: fc3fda29f660/172.17.0.2:38897 2024-11-16T14:57:11,418 INFO [PEWorker-1 {}] procedure.ServerCrashProcedure(339): Splitting WALs pid=57, state=RUNNABLE:SERVER_CRASH_SPLIT_META_LOGS, hasLock=true; ServerCrashProcedure fc3fda29f660,38897,1731769023879, splitWal=true, meta=true, isMeta: true 2024-11-16T14:57:11,421 DEBUG [PEWorker-1 {}] master.MasterWalManager(329): Renamed region directory: hdfs://localhost:42223/user/jenkins/test-data/a8176cb1-2ebb-b530-b3c7-53c9e53c186e/WALs/fc3fda29f660,38897,1731769023879-splitting 2024-11-16T14:57:11,422 INFO [PEWorker-1 {}] master.SplitWALManager(105): fc3fda29f660,38897,1731769023879 WAL count=1, meta=true 2024-11-16T14:57:11,425 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=58, ppid=57, state=RUNNABLE:ACQUIRE_SPLIT_WAL_WORKER, hasLock=false; SplitWALProcedure fc3fda29f660%2C38897%2C1731769023879.meta.1731769025887.meta}] 2024-11-16T14:57:11,425 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(132): Toggle KILL before store update to: false 2024-11-16T14:57:11,425 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update: pid=57, state=WAITING:SERVER_CRASH_DELETE_SPLIT_META_WALS_DIR, hasLock=true; ServerCrashProcedure fc3fda29f660,38897,1731769023879, splitWal=true, meta=true 2024-11-16T14:57:11,425 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-16T14:57:11,425 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update: pid=57, state=WAITING:SERVER_CRASH_DELETE_SPLIT_META_WALS_DIR, hasLock=true; ServerCrashProcedure fc3fda29f660,38897,1731769023879, splitWal=true, meta=true at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1879) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:57:11,435 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-16T14:57:11,500 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true' 2024-11-16T14:57:11,502 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta' 2024-11-16T14:57:11,519 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-16T14:57:11,519 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fc3fda29f660,38897,1731769023879, seqNum=-1] 2024-11-16T14:57:11,519 DEBUG [Async-Client-Retry-Timer-pool-0 {}] ipc.AbstractRpcClient(357): Not trying to connect to fc3fda29f660:38897 this server is in the failed servers list 2024-11-16T14:57:11,520 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=fc3fda29f660,38897,1731769023879, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=fc3fda29f660,38897,1731769023879, seqNum=-1, error=org.apache.hadoop.hbase.ipc.FailedServerException: Call to address=fc3fda29f660:38897 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: fc3fda29f660:38897 2024-11-16T14:57:11,520 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:meta,,1.1588230740, hostname=fc3fda29f660,38897,1731769023879, seqNum=-1 is org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: fc3fda29f660:38897 2024-11-16T14:57:11,520 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:meta,,1.1588230740, hostname=fc3fda29f660,38897,1731769023879, seqNum=-1 from cache 2024-11-16T14:57:11,729 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-16T14:57:11,729 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fc3fda29f660,38897,1731769023879, seqNum=-1] 2024-11-16T14:57:11,730 DEBUG [Async-Client-Retry-Timer-pool-0 {}] ipc.AbstractRpcClient(357): Not trying to connect to fc3fda29f660:38897 this server is in the failed servers list 2024-11-16T14:57:11,731 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=fc3fda29f660,38897,1731769023879, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=fc3fda29f660,38897,1731769023879, seqNum=-1, error=org.apache.hadoop.hbase.ipc.FailedServerException: Call to address=fc3fda29f660:38897 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: fc3fda29f660:38897 2024-11-16T14:57:11,731 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:meta,,1.1588230740, hostname=fc3fda29f660,38897,1731769023879, seqNum=-1 is org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: fc3fda29f660:38897 2024-11-16T14:57:11,731 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:meta,,1.1588230740, hostname=fc3fda29f660,38897,1731769023879, seqNum=-1 from cache 2024-11-16T14:57:12,038 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-16T14:57:12,038 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fc3fda29f660,38897,1731769023879, seqNum=-1] 2024-11-16T14:57:12,039 DEBUG [Async-Client-Retry-Timer-pool-0 {}] ipc.AbstractRpcClient(357): Not trying to connect to fc3fda29f660:38897 this server is in the failed servers list 2024-11-16T14:57:12,039 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=fc3fda29f660,38897,1731769023879, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=fc3fda29f660,38897,1731769023879, seqNum=-1, error=org.apache.hadoop.hbase.ipc.FailedServerException: Call to address=fc3fda29f660:38897 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: fc3fda29f660:38897 2024-11-16T14:57:12,039 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:meta,,1.1588230740, hostname=fc3fda29f660,38897,1731769023879, seqNum=-1 is org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: fc3fda29f660:38897 2024-11-16T14:57:12,039 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:meta,,1.1588230740, hostname=fc3fda29f660,38897,1731769023879, seqNum=-1 from cache 2024-11-16T14:57:12,549 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-16T14:57:12,549 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fc3fda29f660,38897,1731769023879, seqNum=-1] 2024-11-16T14:57:12,549 DEBUG [Async-Client-Retry-Timer-pool-0 {}] ipc.AbstractRpcClient(357): Not trying to connect to fc3fda29f660:38897 this server is in the failed servers list 2024-11-16T14:57:12,550 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=fc3fda29f660,38897,1731769023879, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=fc3fda29f660,38897,1731769023879, seqNum=-1, error=org.apache.hadoop.hbase.ipc.FailedServerException: Call to address=fc3fda29f660:38897 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: fc3fda29f660:38897 2024-11-16T14:57:12,550 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:meta,,1.1588230740, hostname=fc3fda29f660,38897,1731769023879, seqNum=-1 is org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: fc3fda29f660:38897 2024-11-16T14:57:12,550 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:meta,,1.1588230740, hostname=fc3fda29f660,38897,1731769023879, seqNum=-1 from cache 2024-11-16T14:57:13,381 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta 2024-11-16T14:57:13,381 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_meta Metrics about Tables on a single HBase RegionServer 2024-11-16T14:57:13,384 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true 2024-11-16T14:57:13,385 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true Metrics about Tables on a single HBase RegionServer 2024-11-16T14:57:13,387 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-16T14:57:13,387 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint Metrics about HBase RegionObservers 2024-11-16T14:57:13,387 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_master_table_store 2024-11-16T14:57:13,388 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_master_table_store Metrics about Tables on a single HBase RegionServer 2024-11-16T14:57:13,559 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-16T14:57:13,559 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fc3fda29f660,38897,1731769023879, seqNum=-1] 2024-11-16T14:57:13,560 WARN [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.NettyRpcConnection$2(409): Exception encountered while connecting to the server fc3fda29f660:38897 org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: fc3fda29f660/172.17.0.2:38897 Caused by: java.net.ConnectException: finishConnect(..) failed: Connection refused at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.newConnectException0(Errors.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.handleConnectErrno(Errors.java:131) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Socket.finishConnect(Socket.java:359) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.doFinishConnect(AbstractEpollChannel.java:715) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:692) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:57:13,564 WARN [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRpcRetryingCaller(177): Call to fc3fda29f660,38897,1731769023879 for '' in 1588230740 of hbase:meta failed, tries = 6, maxAttempts = 46, timeout = 60000 ms, time elapsed = 2147 ms java.net.ConnectException: Call to address=fc3fda29f660:38897 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: fc3fda29f660/172.17.0.2:38897 at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:220) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:395) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.BufferCallBeforeInitHandler.userEventTriggered(BufferCallBeforeInitHandler.java:106) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:398) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:376) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireUserEventTriggered(AbstractChannelHandlerContext.java:368) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.userEventTriggered(DefaultChannelPipeline.java:1375) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:396) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:376) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireUserEventTriggered(DefaultChannelPipeline.java:862) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcConnection.failInit(NettyRpcConnection.java:210) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcConnection$2.fail(NettyRpcConnection.java:414) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcConnection$2.operationComplete(NettyRpcConnection.java:421) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcConnection$2.operationComplete(NettyRpcConnection.java:389) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:629) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:118) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.fulfillConnectPromise(AbstractEpollChannel.java:679) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:698) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: fc3fda29f660/172.17.0.2:38897 Caused by: java.net.ConnectException: finishConnect(..) failed: Connection refused at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.newConnectException0(Errors.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.handleConnectErrno(Errors.java:131) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Socket.finishConnect(Socket.java:359) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.doFinishConnect(AbstractEpollChannel.java:715) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:692) ~[hbase-shaded-netty-4.1.9.jar:?] ... 7 more 2024-11-16T14:57:13,565 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=fc3fda29f660,38897,1731769023879, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=fc3fda29f660,38897,1731769023879, seqNum=-1, error=java.net.ConnectException: Call to address=fc3fda29f660:38897 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: fc3fda29f660/172.17.0.2:38897 2024-11-16T14:57:13,565 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:meta,,1.1588230740, hostname=fc3fda29f660,38897,1731769023879, seqNum=-1 is java.net.ConnectException: finishConnect(..) failed: Connection refused 2024-11-16T14:57:13,565 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:meta,,1.1588230740, hostname=fc3fda29f660,38897,1731769023879, seqNum=-1 from cache 2024-11-16T14:57:13,565 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.FailedServers(52): Added failed server with address fc3fda29f660:38897 to list caused by org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: fc3fda29f660/172.17.0.2:38897 2024-11-16T14:57:15,588 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-16T14:57:15,589 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fc3fda29f660,38897,1731769023879, seqNum=-1] 2024-11-16T14:57:15,590 WARN [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.NettyRpcConnection$2(409): Exception encountered while connecting to the server fc3fda29f660:38897 org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: fc3fda29f660/172.17.0.2:38897 Caused by: java.net.ConnectException: finishConnect(..) failed: Connection refused at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.newConnectException0(Errors.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.handleConnectErrno(Errors.java:131) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Socket.finishConnect(Socket.java:359) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.doFinishConnect(AbstractEpollChannel.java:715) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:692) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:57:15,591 WARN [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRpcRetryingCaller(177): Call to fc3fda29f660,38897,1731769023879 for '' in 1588230740 of hbase:meta failed, tries = 7, maxAttempts = 46, timeout = 60000 ms, time elapsed = 4177 ms java.net.ConnectException: Call to address=fc3fda29f660:38897 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: fc3fda29f660/172.17.0.2:38897 at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:220) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:395) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.BufferCallBeforeInitHandler.userEventTriggered(BufferCallBeforeInitHandler.java:106) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:398) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:376) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireUserEventTriggered(AbstractChannelHandlerContext.java:368) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.userEventTriggered(DefaultChannelPipeline.java:1375) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:396) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:376) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireUserEventTriggered(DefaultChannelPipeline.java:862) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcConnection.failInit(NettyRpcConnection.java:210) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcConnection$2.fail(NettyRpcConnection.java:414) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcConnection$2.operationComplete(NettyRpcConnection.java:421) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcConnection$2.operationComplete(NettyRpcConnection.java:389) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:629) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:118) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.fulfillConnectPromise(AbstractEpollChannel.java:679) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:698) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: fc3fda29f660/172.17.0.2:38897 Caused by: java.net.ConnectException: finishConnect(..) failed: Connection refused at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.newConnectException0(Errors.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.handleConnectErrno(Errors.java:131) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Socket.finishConnect(Socket.java:359) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.doFinishConnect(AbstractEpollChannel.java:715) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:692) ~[hbase-shaded-netty-4.1.9.jar:?] ... 7 more 2024-11-16T14:57:15,593 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=fc3fda29f660,38897,1731769023879, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=fc3fda29f660,38897,1731769023879, seqNum=-1, error=java.net.ConnectException: Call to address=fc3fda29f660:38897 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: fc3fda29f660/172.17.0.2:38897 2024-11-16T14:57:15,593 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:meta,,1.1588230740, hostname=fc3fda29f660,38897,1731769023879, seqNum=-1 is java.net.ConnectException: finishConnect(..) failed: Connection refused 2024-11-16T14:57:15,593 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:meta,,1.1588230740, hostname=fc3fda29f660,38897,1731769023879, seqNum=-1 from cache 2024-11-16T14:57:15,594 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.FailedServers(52): Added failed server with address fc3fda29f660:38897 to list caused by org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: fc3fda29f660/172.17.0.2:38897 2024-11-16T14:57:19,629 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-16T14:57:19,629 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fc3fda29f660,38897,1731769023879, seqNum=-1] 2024-11-16T14:57:19,630 WARN [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.NettyRpcConnection$2(409): Exception encountered while connecting to the server fc3fda29f660:38897 org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: fc3fda29f660/172.17.0.2:38897 Caused by: java.net.ConnectException: finishConnect(..) failed: Connection refused at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.newConnectException0(Errors.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.handleConnectErrno(Errors.java:131) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Socket.finishConnect(Socket.java:359) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.doFinishConnect(AbstractEpollChannel.java:715) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:692) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:57:19,631 WARN [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRpcRetryingCaller(177): Call to fc3fda29f660,38897,1731769023879 for '' in 1588230740 of hbase:meta failed, tries = 8, maxAttempts = 46, timeout = 60000 ms, time elapsed = 8217 ms java.net.ConnectException: Call to address=fc3fda29f660:38897 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: fc3fda29f660/172.17.0.2:38897 at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:220) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:395) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.BufferCallBeforeInitHandler.userEventTriggered(BufferCallBeforeInitHandler.java:106) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:398) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:376) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireUserEventTriggered(AbstractChannelHandlerContext.java:368) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.userEventTriggered(DefaultChannelPipeline.java:1375) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:396) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:376) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireUserEventTriggered(DefaultChannelPipeline.java:862) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcConnection.failInit(NettyRpcConnection.java:210) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcConnection$2.fail(NettyRpcConnection.java:414) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcConnection$2.operationComplete(NettyRpcConnection.java:421) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcConnection$2.operationComplete(NettyRpcConnection.java:389) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:629) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:118) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.fulfillConnectPromise(AbstractEpollChannel.java:679) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:698) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: fc3fda29f660/172.17.0.2:38897 Caused by: java.net.ConnectException: finishConnect(..) failed: Connection refused at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.newConnectException0(Errors.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.handleConnectErrno(Errors.java:131) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Socket.finishConnect(Socket.java:359) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.doFinishConnect(AbstractEpollChannel.java:715) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:692) ~[hbase-shaded-netty-4.1.9.jar:?] ... 7 more 2024-11-16T14:57:19,632 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=fc3fda29f660,38897,1731769023879, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=fc3fda29f660,38897,1731769023879, seqNum=-1, error=java.net.ConnectException: Call to address=fc3fda29f660:38897 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: fc3fda29f660/172.17.0.2:38897 2024-11-16T14:57:19,632 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:meta,,1.1588230740, hostname=fc3fda29f660,38897,1731769023879, seqNum=-1 is java.net.ConnectException: finishConnect(..) failed: Connection refused 2024-11-16T14:57:19,632 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:meta,,1.1588230740, hostname=fc3fda29f660,38897,1731769023879, seqNum=-1 from cache 2024-11-16T14:57:19,632 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.FailedServers(52): Added failed server with address fc3fda29f660:38897 to list caused by org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: fc3fda29f660/172.17.0.2:38897 2024-11-16T14:57:29,649 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-16T14:57:29,649 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fc3fda29f660,38897,1731769023879, seqNum=-1] 2024-11-16T14:57:29,651 WARN [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.NettyRpcConnection$2(409): Exception encountered while connecting to the server fc3fda29f660:38897 org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: fc3fda29f660/172.17.0.2:38897 Caused by: java.net.ConnectException: finishConnect(..) failed: Connection refused at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.newConnectException0(Errors.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.handleConnectErrno(Errors.java:131) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Socket.finishConnect(Socket.java:359) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.doFinishConnect(AbstractEpollChannel.java:715) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:692) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:57:29,652 WARN [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRpcRetryingCaller(177): Call to fc3fda29f660,38897,1731769023879 for '' in 1588230740 of hbase:meta failed, tries = 9, maxAttempts = 46, timeout = 60000 ms, time elapsed = 18238 ms java.net.ConnectException: Call to address=fc3fda29f660:38897 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: fc3fda29f660/172.17.0.2:38897 at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:220) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:395) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.BufferCallBeforeInitHandler.userEventTriggered(BufferCallBeforeInitHandler.java:106) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:398) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:376) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireUserEventTriggered(AbstractChannelHandlerContext.java:368) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.userEventTriggered(DefaultChannelPipeline.java:1375) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:396) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:376) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireUserEventTriggered(DefaultChannelPipeline.java:862) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcConnection.failInit(NettyRpcConnection.java:210) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcConnection$2.fail(NettyRpcConnection.java:414) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcConnection$2.operationComplete(NettyRpcConnection.java:421) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcConnection$2.operationComplete(NettyRpcConnection.java:389) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:629) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:118) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.fulfillConnectPromise(AbstractEpollChannel.java:679) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:698) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: fc3fda29f660/172.17.0.2:38897 Caused by: java.net.ConnectException: finishConnect(..) failed: Connection refused at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.newConnectException0(Errors.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.handleConnectErrno(Errors.java:131) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Socket.finishConnect(Socket.java:359) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.doFinishConnect(AbstractEpollChannel.java:715) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:692) ~[hbase-shaded-netty-4.1.9.jar:?] ... 7 more 2024-11-16T14:57:29,653 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=fc3fda29f660,38897,1731769023879, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=fc3fda29f660,38897,1731769023879, seqNum=-1, error=java.net.ConnectException: Call to address=fc3fda29f660:38897 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: fc3fda29f660/172.17.0.2:38897 2024-11-16T14:57:29,653 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:meta,,1.1588230740, hostname=fc3fda29f660,38897,1731769023879, seqNum=-1 is java.net.ConnectException: finishConnect(..) failed: Connection refused 2024-11-16T14:57:29,653 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:meta,,1.1588230740, hostname=fc3fda29f660,38897,1731769023879, seqNum=-1 from cache 2024-11-16T14:57:29,653 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.FailedServers(52): Added failed server with address fc3fda29f660:38897 to list caused by org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: fc3fda29f660/172.17.0.2:38897 2024-11-16T14:57:32,164 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-16T14:57:37,023 WARN [BootstrapNodeManager {}] regionserver.BootstrapNodeManager(142): failed to get live region servers from master org.apache.hadoop.hbase.ipc.StoppedRpcClientException: Call to address=fc3fda29f660:43615 failed on local exception: org.apache.hadoop.hbase.ipc.StoppedRpcClientException at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:163) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.BootstrapNodeManager.getFromMaster(BootstrapNodeManager.java:140) ~[classes/:?] at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:539) ~[?:?] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:304) ~[?:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] at --------Future.get--------(Unknown Source) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:237) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:395) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:451) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$RpcChannelImplementation.callMethod(AbstractRpcClient.java:628) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$Stub.getLiveRegionServers(RegionServerStatusProtos.java:17191) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncClusterConnectionImpl.getLiveRegionServers(AsyncClusterConnectionImpl.java:139) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.BootstrapNodeManager.getFromMaster(BootstrapNodeManager.java:140) ~[classes/:?] at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:539) ~[?:?] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:304) ~[?:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.StoppedRpcClientException at org.apache.hadoop.hbase.ipc.AbstractRpcClient.getConnection(AbstractRpcClient.java:366) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:448) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 10 more 2024-11-16T14:57:39,688 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-16T14:57:39,689 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fc3fda29f660,38897,1731769023879, seqNum=-1] 2024-11-16T14:57:39,689 WARN [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.NettyRpcConnection$2(409): Exception encountered while connecting to the server fc3fda29f660:38897 org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: fc3fda29f660/172.17.0.2:38897 Caused by: java.net.ConnectException: finishConnect(..) failed: Connection refused at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.newConnectException0(Errors.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.handleConnectErrno(Errors.java:131) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Socket.finishConnect(Socket.java:359) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.doFinishConnect(AbstractEpollChannel.java:715) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:692) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:57:39,690 WARN [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRpcRetryingCaller(177): Call to fc3fda29f660,38897,1731769023879 for '' in 1588230740 of hbase:meta failed, tries = 10, maxAttempts = 46, timeout = 60000 ms, time elapsed = 28276 ms java.net.ConnectException: Call to address=fc3fda29f660:38897 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: fc3fda29f660/172.17.0.2:38897 at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:220) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:395) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.BufferCallBeforeInitHandler.userEventTriggered(BufferCallBeforeInitHandler.java:106) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:398) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:376) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireUserEventTriggered(AbstractChannelHandlerContext.java:368) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.userEventTriggered(DefaultChannelPipeline.java:1375) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:396) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:376) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireUserEventTriggered(DefaultChannelPipeline.java:862) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcConnection.failInit(NettyRpcConnection.java:210) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcConnection$2.fail(NettyRpcConnection.java:414) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcConnection$2.operationComplete(NettyRpcConnection.java:421) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcConnection$2.operationComplete(NettyRpcConnection.java:389) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:629) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:118) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.fulfillConnectPromise(AbstractEpollChannel.java:679) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:698) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: fc3fda29f660/172.17.0.2:38897 Caused by: java.net.ConnectException: finishConnect(..) failed: Connection refused at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.newConnectException0(Errors.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.handleConnectErrno(Errors.java:131) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Socket.finishConnect(Socket.java:359) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.doFinishConnect(AbstractEpollChannel.java:715) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:692) ~[hbase-shaded-netty-4.1.9.jar:?] ... 7 more 2024-11-16T14:57:39,691 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=fc3fda29f660,38897,1731769023879, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=fc3fda29f660,38897,1731769023879, seqNum=-1, error=java.net.ConnectException: Call to address=fc3fda29f660:38897 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: fc3fda29f660/172.17.0.2:38897 2024-11-16T14:57:39,691 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:meta,,1.1588230740, hostname=fc3fda29f660,38897,1731769023879, seqNum=-1 is java.net.ConnectException: finishConnect(..) failed: Connection refused 2024-11-16T14:57:39,691 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:meta,,1.1588230740, hostname=fc3fda29f660,38897,1731769023879, seqNum=-1 from cache 2024-11-16T14:57:39,691 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.FailedServers(52): Added failed server with address fc3fda29f660:38897 to list caused by org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: fc3fda29f660/172.17.0.2:38897 2024-11-16T14:57:46,411 INFO [master/fc3fda29f660:0.Chore.1 {}] migrate.RollingUpgradeChore(116): There is no table to migrate StoreFileTracker! 2024-11-16T14:57:46,411 INFO [master/fc3fda29f660:0.Chore.1 {}] migrate.RollingUpgradeChore(85): All Rolling-Upgrade tasks are complete, shutdown RollingUpgradeChore! 2024-11-16T14:57:49,739 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-16T14:57:49,739 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fc3fda29f660,38897,1731769023879, seqNum=-1] 2024-11-16T14:57:49,740 WARN [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.NettyRpcConnection$2(409): Exception encountered while connecting to the server fc3fda29f660:38897 org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: fc3fda29f660/172.17.0.2:38897 Caused by: java.net.ConnectException: finishConnect(..) failed: Connection refused at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.newConnectException0(Errors.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.handleConnectErrno(Errors.java:131) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Socket.finishConnect(Socket.java:359) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.doFinishConnect(AbstractEpollChannel.java:715) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:692) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:57:49,740 WARN [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRpcRetryingCaller(177): Call to fc3fda29f660,38897,1731769023879 for '' in 1588230740 of hbase:meta failed, tries = 11, maxAttempts = 46, timeout = 60000 ms, time elapsed = 38326 ms java.net.ConnectException: Call to address=fc3fda29f660:38897 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: fc3fda29f660/172.17.0.2:38897 at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:220) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:395) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.BufferCallBeforeInitHandler.userEventTriggered(BufferCallBeforeInitHandler.java:106) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:398) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:376) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireUserEventTriggered(AbstractChannelHandlerContext.java:368) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.userEventTriggered(DefaultChannelPipeline.java:1375) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:396) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:376) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireUserEventTriggered(DefaultChannelPipeline.java:862) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcConnection.failInit(NettyRpcConnection.java:210) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcConnection$2.fail(NettyRpcConnection.java:414) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcConnection$2.operationComplete(NettyRpcConnection.java:421) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcConnection$2.operationComplete(NettyRpcConnection.java:389) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:629) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:118) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.fulfillConnectPromise(AbstractEpollChannel.java:679) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:698) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: fc3fda29f660/172.17.0.2:38897 Caused by: java.net.ConnectException: finishConnect(..) failed: Connection refused at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.newConnectException0(Errors.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.handleConnectErrno(Errors.java:131) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Socket.finishConnect(Socket.java:359) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.doFinishConnect(AbstractEpollChannel.java:715) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:692) ~[hbase-shaded-netty-4.1.9.jar:?] ... 7 more 2024-11-16T14:57:49,740 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=fc3fda29f660,38897,1731769023879, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=fc3fda29f660,38897,1731769023879, seqNum=-1, error=java.net.ConnectException: Call to address=fc3fda29f660:38897 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: fc3fda29f660/172.17.0.2:38897 2024-11-16T14:57:49,740 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:meta,,1.1588230740, hostname=fc3fda29f660,38897,1731769023879, seqNum=-1 is java.net.ConnectException: finishConnect(..) failed: Connection refused 2024-11-16T14:57:49,740 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:meta,,1.1588230740, hostname=fc3fda29f660,38897,1731769023879, seqNum=-1 from cache 2024-11-16T14:57:49,740 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.FailedServers(52): Added failed server with address fc3fda29f660:38897 to list caused by org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: fc3fda29f660/172.17.0.2:38897 2024-11-16T14:57:55,766 WARN [RS:0;fc3fda29f660:39089 {}] monitoring.TaskMonitor(166): Status Processing ServerCrashProcedure of fc3fda29f660,38897,1731769023879: status=status unset, state=RUNNING, startTime=1731769031334, completionTime=-1 appears to have been leaked 2024-11-16T14:57:59,850 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-16T14:57:59,850 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fc3fda29f660,38897,1731769023879, seqNum=-1] 2024-11-16T14:57:59,853 WARN [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.NettyRpcConnection$2(409): Exception encountered while connecting to the server fc3fda29f660:38897 org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: fc3fda29f660/172.17.0.2:38897 Caused by: java.net.ConnectException: finishConnect(..) failed: Connection refused at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.newConnectException0(Errors.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.handleConnectErrno(Errors.java:131) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Socket.finishConnect(Socket.java:359) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.doFinishConnect(AbstractEpollChannel.java:715) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:692) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:57:59,854 WARN [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRpcRetryingCaller(177): Call to fc3fda29f660,38897,1731769023879 for '' in 1588230740 of hbase:meta failed, tries = 12, maxAttempts = 46, timeout = 60000 ms, time elapsed = 48440 ms java.net.ConnectException: Call to address=fc3fda29f660:38897 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: fc3fda29f660/172.17.0.2:38897 at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:220) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:395) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.BufferCallBeforeInitHandler.userEventTriggered(BufferCallBeforeInitHandler.java:106) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:398) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:376) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireUserEventTriggered(AbstractChannelHandlerContext.java:368) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.userEventTriggered(DefaultChannelPipeline.java:1375) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:396) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:376) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireUserEventTriggered(DefaultChannelPipeline.java:862) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcConnection.failInit(NettyRpcConnection.java:210) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcConnection$2.fail(NettyRpcConnection.java:414) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcConnection$2.operationComplete(NettyRpcConnection.java:421) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcConnection$2.operationComplete(NettyRpcConnection.java:389) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:629) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:118) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.fulfillConnectPromise(AbstractEpollChannel.java:679) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:698) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: fc3fda29f660/172.17.0.2:38897 Caused by: java.net.ConnectException: finishConnect(..) failed: Connection refused at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.newConnectException0(Errors.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.handleConnectErrno(Errors.java:131) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Socket.finishConnect(Socket.java:359) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.doFinishConnect(AbstractEpollChannel.java:715) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:692) ~[hbase-shaded-netty-4.1.9.jar:?] ... 7 more 2024-11-16T14:57:59,855 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=fc3fda29f660,38897,1731769023879, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=fc3fda29f660,38897,1731769023879, seqNum=-1, error=java.net.ConnectException: Call to address=fc3fda29f660:38897 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: fc3fda29f660/172.17.0.2:38897 2024-11-16T14:57:59,856 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:meta,,1.1588230740, hostname=fc3fda29f660,38897,1731769023879, seqNum=-1 is java.net.ConnectException: finishConnect(..) failed: Connection refused 2024-11-16T14:57:59,856 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:meta,,1.1588230740, hostname=fc3fda29f660,38897,1731769023879, seqNum=-1 from cache 2024-11-16T14:57:59,856 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.FailedServers(52): Added failed server with address fc3fda29f660:38897 to list caused by org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: fc3fda29f660/172.17.0.2:38897 2024-11-16T14:58:02,164 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-16T14:58:05,283 INFO [regionserver/fc3fda29f660:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: ExecutorStatusChore was stopped 2024-11-16T14:58:07,412 DEBUG [master/fc3fda29f660:0.Chore.1 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 5bb22144956dcd43c9b1d957232b11e7 changed from -1.0 to 0.0, refreshing cache 2024-11-16T14:58:07,413 DEBUG [master/fc3fda29f660:0.Chore.1 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region dde7f54607f11c5fb4271e361e82dabd changed from -1.0 to 0.0, refreshing cache 2024-11-16T14:58:07,413 DEBUG [master/fc3fda29f660:0.Chore.1 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region b4f73326a2a6a279d503e557c6ed452f changed from -1.0 to 0.0, refreshing cache 2024-11-16T14:58:07,414 DEBUG [master/fc3fda29f660:0.Chore.1 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 994ca1526cf6c4838e65f85444853f1a changed from -1.0 to 0.0, refreshing cache 2024-11-16T14:58:07,415 DEBUG [master/fc3fda29f660:0.Chore.1 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 18f03e2509a3a89e4a69ba87fad73940 changed from -1.0 to 0.0, refreshing cache 2024-11-16T14:58:07,415 DEBUG [master/fc3fda29f660:0.Chore.1 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 7f11f79823720d664105d8643dfa862d changed from -1.0 to 0.0, refreshing cache 2024-11-16T14:58:07,415 DEBUG [master/fc3fda29f660:0.Chore.1 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 62a6d124ad9d161782c0632102c766de changed from -1.0 to 0.0, refreshing cache 2024-11-16T14:58:07,415 DEBUG [master/fc3fda29f660:0.Chore.1 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 2f41a8c996224c818e917389bd3f5141 changed from -1.0 to 0.0, refreshing cache 2024-11-16T14:58:07,416 DEBUG [master/fc3fda29f660:0.Chore.1 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region c3dde5f5f668d914608a2987d4f4c001 changed from -1.0 to 0.0, refreshing cache 2024-11-16T14:58:07,416 DEBUG [master/fc3fda29f660:0.Chore.1 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region a5db9431e22152bd2a82e84504d2879a changed from -1.0 to 0.0, refreshing cache 2024-11-16T14:58:07,416 DEBUG [master/fc3fda29f660:0.Chore.1 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region dcd6414ddebad66e87ab37a6dba924f3 changed from -1.0 to 0.0, refreshing cache 2024-11-16T14:58:07,416 DEBUG [master/fc3fda29f660:0.Chore.1 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 1d97df67e5110b0fe1a1e4b7c1341cbb changed from -1.0 to 0.0, refreshing cache 2024-11-16T14:58:07,416 DEBUG [master/fc3fda29f660:0.Chore.1 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 266565e60d7419a2a30ceec1b0f6519b changed from -1.0 to 0.0, refreshing cache 2024-11-16T14:58:07,417 DEBUG [master/fc3fda29f660:0.Chore.1 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 299131947f15f9c44bcfe22cd6623096 changed from -1.0 to 0.0, refreshing cache 2024-11-16T14:58:07,417 DEBUG [master/fc3fda29f660:0.Chore.1 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region b364a006589fe0d2474ffa45692a6efc changed from -1.0 to 0.0, refreshing cache 2024-11-16T14:58:07,417 DEBUG [master/fc3fda29f660:0.Chore.1 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 35ef9a7f35bb0b08ec1d53b82d3f9777 changed from -1.0 to 0.0, refreshing cache 2024-11-16T14:58:07,417 DEBUG [master/fc3fda29f660:0.Chore.1 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region dc4fcaece350cd66f6bf88cfbdff6f15 changed from -1.0 to 0.0, refreshing cache 2024-11-16T14:58:11,420 WARN [Time-limited test {}] procedure.MasterProcedureTestingUtility$3(130): Failed to load meta org.apache.hadoop.hbase.client.RetriesExhaustedException: Failed after attempts=13, exceptions: 2024-11-16T14:57:11.417Z, java.net.ConnectException: Call to address=fc3fda29f660:38897 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: fc3fda29f660/172.17.0.2:38897 2024-11-16T14:57:11.520Z, org.apache.hadoop.hbase.ipc.FailedServerException: Call to address=fc3fda29f660:38897 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: fc3fda29f660:38897 2024-11-16T14:57:11.731Z, org.apache.hadoop.hbase.ipc.FailedServerException: Call to address=fc3fda29f660:38897 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: fc3fda29f660:38897 2024-11-16T14:57:12.039Z, org.apache.hadoop.hbase.ipc.FailedServerException: Call to address=fc3fda29f660:38897 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: fc3fda29f660:38897 2024-11-16T14:57:12.550Z, org.apache.hadoop.hbase.ipc.FailedServerException: Call to address=fc3fda29f660:38897 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: fc3fda29f660:38897 2024-11-16T14:57:13.565Z, java.net.ConnectException: Call to address=fc3fda29f660:38897 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: fc3fda29f660/172.17.0.2:38897 2024-11-16T14:57:15.593Z, java.net.ConnectException: Call to address=fc3fda29f660:38897 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: fc3fda29f660/172.17.0.2:38897 2024-11-16T14:57:19.632Z, java.net.ConnectException: Call to address=fc3fda29f660:38897 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: fc3fda29f660/172.17.0.2:38897 2024-11-16T14:57:29.653Z, java.net.ConnectException: Call to address=fc3fda29f660:38897 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: fc3fda29f660/172.17.0.2:38897 2024-11-16T14:57:39.691Z, java.net.ConnectException: Call to address=fc3fda29f660:38897 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: fc3fda29f660/172.17.0.2:38897 2024-11-16T14:57:49.740Z, java.net.ConnectException: Call to address=fc3fda29f660:38897 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: fc3fda29f660/172.17.0.2:38897 2024-11-16T14:57:59.856Z, java.net.ConnectException: Call to address=fc3fda29f660:38897 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: fc3fda29f660/172.17.0.2:38897 at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:163) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncTableResultScanner.next(AsyncTableResultScanner.java:173) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.MetaTableAccessor.scanMeta(MetaTableAccessor.java:493) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.MetaTableAccessor.scanMeta(MetaTableAccessor.java:461) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.MetaTableAccessor.scanMeta(MetaTableAccessor.java:422) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.MetaTableAccessor.fullScanRegions(MetaTableAccessor.java:108) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionStateStore.visitMeta(RegionStateStore.java:108) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.AssignmentManager.loadMeta(AssignmentManager.java:1907) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.AssignmentManager.joinCluster(AssignmentManager.java:1769) ~[classes/:?] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility$3.call(MasterProcedureTestingUtility.java:126) ~[test-classes/:?] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility$3.call(MasterProcedureTestingUtility.java:121) ~[test-classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:143) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT-tests.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT-tests.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.testRecoveryAndDoubleExecution(MasterProcedureTestingUtility.java:477) ~[test-classes/:?] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.testRecoveryAndDoubleExecution(MasterProcedureTestingUtility.java:486) ~[test-classes/:?] at org.apache.hadoop.hbase.master.procedure.TestSCPBase.testRecoveryAndDoubleExecution(TestSCPBase.java:114) ~[test-classes/:?] at org.apache.hadoop.hbase.master.procedure.TestSCPWithMeta.testRecoveryAndDoubleExecutionOnRsWithMeta(TestSCPWithMeta.java:36) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.completeExceptionally(AsyncRpcRetryingCaller.java:109) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.doCall(AsyncSingleRequestRpcRetryingCaller.java:102) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.lambda$tryScheduleRetry$1(AsyncRpcRetryingCaller.java:139) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelTimeout.run(HashedWheelTimer.java:713) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.ImmediateExecutor.execute(ImmediateExecutor.java:34) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelTimeout.expire(HashedWheelTimer.java:701) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelBucket.expireTimeouts(HashedWheelTimer.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:501) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.net.ConnectException: Call to address=fc3fda29f660:38897 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: fc3fda29f660/172.17.0.2:38897 at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:220) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:395) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.BufferCallBeforeInitHandler.userEventTriggered(BufferCallBeforeInitHandler.java:106) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:398) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:376) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireUserEventTriggered(AbstractChannelHandlerContext.java:368) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.userEventTriggered(DefaultChannelPipeline.java:1375) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:396) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:376) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireUserEventTriggered(DefaultChannelPipeline.java:862) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcConnection.failInit(NettyRpcConnection.java:210) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcConnection$2.fail(NettyRpcConnection.java:414) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcConnection$2.operationComplete(NettyRpcConnection.java:421) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcConnection$2.operationComplete(NettyRpcConnection.java:389) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:629) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:118) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.fulfillConnectPromise(AbstractEpollChannel.java:679) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:698) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] ... 1 more Caused by: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: fc3fda29f660/172.17.0.2:38897 Caused by: java.net.ConnectException: finishConnect(..) failed: Connection refused at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.newConnectException0(Errors.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.handleConnectErrno(Errors.java:131) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Socket.finishConnect(Socket.java:359) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.doFinishConnect(AbstractEpollChannel.java:715) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:692) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] ... 1 more 2024-11-16T14:58:11,421 INFO [Time-limited test {}] procedure.MasterProcedureTestingUtility(473): Restart 1 exec state=pid=57, state=WAITING:SERVER_CRASH_DELETE_SPLIT_META_WALS_DIR, hasLock=true; ServerCrashProcedure fc3fda29f660,38897,1731769023879, splitWal=true, meta=true 2024-11-16T14:58:11,421 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-16T14:58:11,422 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-16T14:58:11,422 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-16T14:58:11,422 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-16T14:58:11,422 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-16T14:58:11,422 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-16T14:58:11,422 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-16T14:58:11,423 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-16T14:58:11,423 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-16T14:58:11,423 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=43615 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:630) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:58:11,426 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=43615 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:630) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:58:11,427 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=57, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-16T14:58:11,427 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-16T14:58:11,429 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true 2024-11-16T14:58:11,429 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=57, state=RUNNABLE:SERVER_CRASH_SPLIT_META_LOGS, hasLock=false; ServerCrashProcedure fc3fda29f660,38897,1731769023879, splitWal=true, meta=true 2024-11-16T14:58:11,429 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=57, state=RUNNABLE:SERVER_CRASH_SPLIT_META_LOGS, hasLock=false; ServerCrashProcedure fc3fda29f660,38897,1731769023879, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-16T14:58:11,430 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 6 msec 2024-11-16T14:58:11,431 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=fc3fda29f660,38897,1731769023879, regionLocation=fc3fda29f660,38897,1731769023879, openSeqNum=2 2024-11-16T14:58:11,431 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=fc3fda29f660,38897,1731769023879, table=hbase:meta, region=1588230740 2024-11-16T14:58:11,431 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as fc3fda29f660,38897,1731769023879, state=OPEN 2024-11-16T14:58:11,505 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43615-0x101456fc9c70000, quorum=127.0.0.1:63224, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-16T14:58:11,505 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42985-0x101456fc9c70003, quorum=127.0.0.1:63224, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-16T14:58:11,505 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39089-0x101456fc9c70001, quorum=127.0.0.1:63224, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-16T14:58:11,505 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-16T14:58:11,505 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-16T14:58:11,505 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-16T14:58:11,506 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731769121506 2024-11-16T14:58:11,506 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-16T14:58:11,507 INFO [PEWorker-1 {}] procedure.ServerCrashProcedure(339): Splitting WALs pid=57, state=RUNNABLE:SERVER_CRASH_SPLIT_META_LOGS, hasLock=true; ServerCrashProcedure fc3fda29f660,38897,1731769023879, splitWal=true, meta=true, isMeta: true 2024-11-16T14:58:11,507 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-16T14:58:11,507 DEBUG [Time-limited test {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fc3fda29f660,38897,1731769023879, seqNum=-1] 2024-11-16T14:58:11,508 WARN [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.NettyRpcConnection$2(409): Exception encountered while connecting to the server fc3fda29f660:38897 org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: fc3fda29f660/172.17.0.2:38897 Caused by: java.net.ConnectException: finishConnect(..) failed: Connection refused at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.newConnectException0(Errors.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.handleConnectErrno(Errors.java:131) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Socket.finishConnect(Socket.java:359) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.doFinishConnect(AbstractEpollChannel.java:715) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:692) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:58:11,509 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=fc3fda29f660,38897,1731769023879, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=fc3fda29f660,38897,1731769023879, seqNum=-1, error=java.net.ConnectException: Call to address=fc3fda29f660:38897 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: fc3fda29f660/172.17.0.2:38897 2024-11-16T14:58:11,509 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:meta,,1.1588230740, hostname=fc3fda29f660,38897,1731769023879, seqNum=-1 is java.net.ConnectException: finishConnect(..) failed: Connection refused 2024-11-16T14:58:11,509 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:meta,,1.1588230740, hostname=fc3fda29f660,38897,1731769023879, seqNum=-1 from cache 2024-11-16T14:58:11,509 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.FailedServers(52): Added failed server with address fc3fda29f660:38897 to list caused by org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: fc3fda29f660/172.17.0.2:38897 2024-11-16T14:58:11,510 INFO [PEWorker-1 {}] master.SplitWALManager(105): fc3fda29f660,38897,1731769023879 WAL count=1, meta=true 2024-11-16T14:58:11,510 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=58, ppid=57, state=RUNNABLE:ACQUIRE_SPLIT_WAL_WORKER, hasLock=false; SplitWALProcedure fc3fda29f660%2C38897%2C1731769023879.meta.1731769025887.meta}] 2024-11-16T14:58:11,510 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(132): Toggle KILL before store update to: true 2024-11-16T14:58:11,517 DEBUG [PEWorker-1 {}] master.SplitWALManager(158): Acquired split WAL worker=fc3fda29f660,42985,1731769023951 2024-11-16T14:58:11,518 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=59, ppid=58, state=RUNNABLE, hasLock=false; SplitWALRemoteProcedure fc3fda29f660%2C38897%2C1731769023879.meta.1731769025887.meta, worker=fc3fda29f660,42985,1731769023951}] 2024-11-16T14:58:11,619 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-16T14:58:11,619 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fc3fda29f660,38897,1731769023879, seqNum=-1] 2024-11-16T14:58:11,619 DEBUG [Async-Client-Retry-Timer-pool-0 {}] ipc.AbstractRpcClient(357): Not trying to connect to fc3fda29f660:38897 this server is in the failed servers list 2024-11-16T14:58:11,620 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=fc3fda29f660,38897,1731769023879, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=fc3fda29f660,38897,1731769023879, seqNum=-1, error=org.apache.hadoop.hbase.ipc.FailedServerException: Call to address=fc3fda29f660:38897 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: fc3fda29f660:38897 2024-11-16T14:58:11,620 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:meta,,1.1588230740, hostname=fc3fda29f660,38897,1731769023879, seqNum=-1 is org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: fc3fda29f660:38897 2024-11-16T14:58:11,620 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:meta,,1.1588230740, hostname=fc3fda29f660,38897,1731769023879, seqNum=-1 from cache 2024-11-16T14:58:11,681 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42985 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SplitWALCallable, pid=59 2024-11-16T14:58:11,705 INFO [RS_LOG_REPLAY_OPS-regionserver/fc3fda29f660:0-0 {event_type=RS_LOG_REPLAY, pid=59}] wal.WALSplitter(299): Splitting hdfs://localhost:42223/user/jenkins/test-data/a8176cb1-2ebb-b530-b3c7-53c9e53c186e/WALs/fc3fda29f660,38897,1731769023879-splitting/fc3fda29f660%2C38897%2C1731769023879.meta.1731769025887.meta, size=52.9 K (54214bytes) 2024-11-16T14:58:11,705 DEBUG [RS_LOG_REPLAY_OPS-regionserver/fc3fda29f660:0-0 {event_type=RS_LOG_REPLAY, pid=59}] util.RecoverLeaseFSUtils(47): Initialize RecoverLeaseFSUtils 2024-11-16T14:58:11,706 DEBUG [RS_LOG_REPLAY_OPS-regionserver/fc3fda29f660:0-0 {event_type=RS_LOG_REPLAY, pid=59}] util.RecoverLeaseFSUtils(59): set recoverLeaseMethod to org.apache.hadoop.fs.LeaseRecoverable.recoverLease() 2024-11-16T14:58:11,706 INFO [RS_LOG_REPLAY_OPS-regionserver/fc3fda29f660:0-0 {event_type=RS_LOG_REPLAY, pid=59}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:42223/user/jenkins/test-data/a8176cb1-2ebb-b530-b3c7-53c9e53c186e/WALs/fc3fda29f660,38897,1731769023879-splitting/fc3fda29f660%2C38897%2C1731769023879.meta.1731769025887.meta 2024-11-16T14:58:11,710 INFO [RS_LOG_REPLAY_OPS-regionserver/fc3fda29f660:0-0 {event_type=RS_LOG_REPLAY, pid=59}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:42223/user/jenkins/test-data/a8176cb1-2ebb-b530-b3c7-53c9e53c186e/WALs/fc3fda29f660,38897,1731769023879-splitting/fc3fda29f660%2C38897%2C1731769023879.meta.1731769025887.meta after 3ms 2024-11-16T14:58:11,717 INFO [RS_LOG_REPLAY_OPS-regionserver/fc3fda29f660:0-0 {event_type=RS_LOG_REPLAY, pid=59}] wal.WALSplitter(310): Open hdfs://localhost:42223/user/jenkins/test-data/a8176cb1-2ebb-b530-b3c7-53c9e53c186e/WALs/fc3fda29f660,38897,1731769023879-splitting/fc3fda29f660%2C38897%2C1731769023879.meta.1731769025887.meta took 13ms 2024-11-16T14:58:11,724 DEBUG [RS_LOG_REPLAY_OPS-regionserver/fc3fda29f660:0-0 {event_type=RS_LOG_REPLAY, pid=59}] wal.WALSplitter(352): Last flushed sequenceid for 1588230740: last_flushed_sequence_id: 18446744073709551615 2024-11-16T14:58:11,730 INFO [RS_LOG_REPLAY_OPS-regionserver/fc3fda29f660:0-0-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000004-fc3fda29f660%2C38897%2C1731769023879.meta.1731769025887.meta.temp 2024-11-16T14:58:11,734 DEBUG [RS_LOG_REPLAY_OPS-regionserver/fc3fda29f660:0-0 {event_type=RS_LOG_REPLAY, pid=59}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:42223/user/jenkins/test-data/a8176cb1-2ebb-b530-b3c7-53c9e53c186e/WALs/fc3fda29f660,38897,1731769023879-splitting/fc3fda29f660%2C38897%2C1731769023879.meta.1731769025887.meta so closing down 2024-11-16T14:58:11,734 DEBUG [RS_LOG_REPLAY_OPS-regionserver/fc3fda29f660:0-0 {event_type=RS_LOG_REPLAY, pid=59}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-16T14:58:11,734 INFO [RS_LOG_REPLAY_OPS-regionserver/fc3fda29f660:0-0-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:42223/user/jenkins/test-data/a8176cb1-2ebb-b530-b3c7-53c9e53c186e/data/hbase/meta/1588230740/recovered.edits/0000000000000000004-fc3fda29f660%2C38897%2C1731769023879.meta.1731769025887.meta.temp 2024-11-16T14:58:11,738 INFO [RS_LOG_REPLAY_OPS-regionserver/fc3fda29f660:0-0 {event_type=RS_LOG_REPLAY, pid=59}] wal.OutputSink(145): 3 split writer threads finished 2024-11-16T14:58:11,749 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40641 is added to blk_1073741864_1040 (size=53973) 2024-11-16T14:58:11,749 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36531 is added to blk_1073741864_1040 (size=53973) 2024-11-16T14:58:11,749 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41671 is added to blk_1073741864_1040 (size=53973) 2024-11-16T14:58:11,752 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:42223/user/jenkins/test-data/a8176cb1-2ebb-b530-b3c7-53c9e53c186e/data/hbase/meta/1588230740/recovered.edits/0000000000000000004-fc3fda29f660%2C38897%2C1731769023879.meta.1731769025887.meta.temp (wrote 57 edits, skipped 0 edits in 3 ms) 2024-11-16T14:58:11,755 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:42223/user/jenkins/test-data/a8176cb1-2ebb-b530-b3c7-53c9e53c186e/data/hbase/meta/1588230740/recovered.edits/0000000000000000004-fc3fda29f660%2C38897%2C1731769023879.meta.1731769025887.meta.temp to hdfs://localhost:42223/user/jenkins/test-data/a8176cb1-2ebb-b530-b3c7-53c9e53c186e/data/hbase/meta/1588230740/recovered.edits/0000000000000000060 2024-11-16T14:58:11,758 INFO [RS_LOG_REPLAY_OPS-regionserver/fc3fda29f660:0-0 {event_type=RS_LOG_REPLAY, pid=59}] wal.WALSplitter(425): Processed 58 edits across 1 Regions in 37 ms; skipped=1; WAL=hdfs://localhost:42223/user/jenkins/test-data/a8176cb1-2ebb-b530-b3c7-53c9e53c186e/WALs/fc3fda29f660,38897,1731769023879-splitting/fc3fda29f660%2C38897%2C1731769023879.meta.1731769025887.meta, size=52.9 K, length=54214, corrupted=false, cancelled=false 2024-11-16T14:58:11,758 DEBUG [RS_LOG_REPLAY_OPS-regionserver/fc3fda29f660:0-0 {event_type=RS_LOG_REPLAY, pid=59}] wal.WALSplitter(428): Completed split of hdfs://localhost:42223/user/jenkins/test-data/a8176cb1-2ebb-b530-b3c7-53c9e53c186e/WALs/fc3fda29f660,38897,1731769023879-splitting/fc3fda29f660%2C38897%2C1731769023879.meta.1731769025887.meta, journal: Splitting hdfs://localhost:42223/user/jenkins/test-data/a8176cb1-2ebb-b530-b3c7-53c9e53c186e/WALs/fc3fda29f660,38897,1731769023879-splitting/fc3fda29f660%2C38897%2C1731769023879.meta.1731769025887.meta, size=52.9 K (54214bytes) at 1731769091705Finishing writing output for hdfs://localhost:42223/user/jenkins/test-data/a8176cb1-2ebb-b530-b3c7-53c9e53c186e/WALs/fc3fda29f660,38897,1731769023879-splitting/fc3fda29f660%2C38897%2C1731769023879.meta.1731769025887.meta so closing down at 1731769091734 (+29 ms)Creating recovered edits writer path=hdfs://localhost:42223/user/jenkins/test-data/a8176cb1-2ebb-b530-b3c7-53c9e53c186e/data/hbase/meta/1588230740/recovered.edits/0000000000000000004-fc3fda29f660%2C38897%2C1731769023879.meta.1731769025887.meta.temp at 17317690917343 split writer threads finished at 1731769091739 (+5 ms)Closed recovered edits writer path=hdfs://localhost:42223/user/jenkins/test-data/a8176cb1-2ebb-b530-b3c7-53c9e53c186e/data/hbase/meta/1588230740/recovered.edits/0000000000000000004-fc3fda29f660%2C38897%2C1731769023879.meta.1731769025887.meta.temp (wrote 57 edits, skipped 0 edits in 3 ms) at 1731769091752 (+13 ms)Rename recovered edits hdfs://localhost:42223/user/jenkins/test-data/a8176cb1-2ebb-b530-b3c7-53c9e53c186e/data/hbase/meta/1588230740/recovered.edits/0000000000000000004-fc3fda29f660%2C38897%2C1731769023879.meta.1731769025887.meta.temp to hdfs://localhost:42223/user/jenkins/test-data/a8176cb1-2ebb-b530-b3c7-53c9e53c186e/data/hbase/meta/1588230740/recovered.edits/0000000000000000060 at 1731769091755 (+3 ms)Processed 58 edits across 1 Regions in 37 ms; skipped=1; WAL=hdfs://localhost:42223/user/jenkins/test-data/a8176cb1-2ebb-b530-b3c7-53c9e53c186e/WALs/fc3fda29f660,38897,1731769023879-splitting/fc3fda29f660%2C38897%2C1731769023879.meta.1731769025887.meta, size=52.9 K, length=54214, corrupted=false, cancelled=false at 1731769091758 (+3 ms) 2024-11-16T14:58:11,758 DEBUG [RS_LOG_REPLAY_OPS-regionserver/fc3fda29f660:0-0 {event_type=RS_LOG_REPLAY, pid=59}] regionserver.SplitLogWorker(218): Done splitting WAL hdfs://localhost:42223/user/jenkins/test-data/a8176cb1-2ebb-b530-b3c7-53c9e53c186e/WALs/fc3fda29f660,38897,1731769023879-splitting/fc3fda29f660%2C38897%2C1731769023879.meta.1731769025887.meta 2024-11-16T14:58:11,760 DEBUG [RS_LOG_REPLAY_OPS-regionserver/fc3fda29f660:0-0 {event_type=RS_LOG_REPLAY, pid=59}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=59 2024-11-16T14:58:11,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43615 {}] master.HMaster(4169): Remote procedure done, pid=59 2024-11-16T14:58:11,769 INFO [PEWorker-1 {}] wal.WALSplitUtil(143): Moved hdfs://localhost:42223/user/jenkins/test-data/a8176cb1-2ebb-b530-b3c7-53c9e53c186e/WALs/fc3fda29f660,38897,1731769023879-splitting/fc3fda29f660%2C38897%2C1731769023879.meta.1731769025887.meta to hdfs://localhost:42223/user/jenkins/test-data/a8176cb1-2ebb-b530-b3c7-53c9e53c186e/oldWALs 2024-11-16T14:58:11,773 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=59, resume processing ppid=58 2024-11-16T14:58:11,773 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure fc3fda29f660%2C38897%2C1731769023879.meta.1731769025887.meta, worker=fc3fda29f660,42985,1731769023951 in 251 msec 2024-11-16T14:58:11,775 DEBUG [PEWorker-1 {}] master.SplitWALManager(172): Release split WAL worker=fc3fda29f660,42985,1731769023951 2024-11-16T14:58:11,778 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=58, resume processing ppid=57 2024-11-16T14:58:11,778 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure fc3fda29f660%2C38897%2C1731769023879.meta.1731769025887.meta, worker=fc3fda29f660,42985,1731769023951 in 265 msec 2024-11-16T14:58:11,780 INFO [PEWorker-1 {}] master.SplitLogManager(171): hdfs://localhost:42223/user/jenkins/test-data/a8176cb1-2ebb-b530-b3c7-53c9e53c186e/WALs/fc3fda29f660,38897,1731769023879-splitting dir is empty, no logs to split. 2024-11-16T14:58:11,780 INFO [PEWorker-1 {}] master.SplitWALManager(105): fc3fda29f660,38897,1731769023879 WAL count=0, meta=true 2024-11-16T14:58:11,780 DEBUG [PEWorker-1 {}] procedure.ServerCrashProcedure(329): Check if fc3fda29f660,38897,1731769023879 WAL splitting is done? wals=0, meta=true 2024-11-16T14:58:11,780 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(132): Toggle KILL before store update to: false 2024-11-16T14:58:11,780 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update: pid=57, state=RUNNABLE:SERVER_CRASH_ASSIGN_META, hasLock=true; ServerCrashProcedure fc3fda29f660,38897,1731769023879, splitWal=true, meta=true 2024-11-16T14:58:11,780 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-16T14:58:11,780 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update: pid=57, state=RUNNABLE:SERVER_CRASH_ASSIGN_META, hasLock=true; ServerCrashProcedure fc3fda29f660,38897,1731769023879, splitWal=true, meta=true at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1879) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:58:11,828 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-16T14:58:11,828 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fc3fda29f660,38897,1731769023879, seqNum=-1] 2024-11-16T14:58:11,829 DEBUG [Async-Client-Retry-Timer-pool-0 {}] ipc.AbstractRpcClient(357): Not trying to connect to fc3fda29f660:38897 this server is in the failed servers list 2024-11-16T14:58:11,829 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=fc3fda29f660,38897,1731769023879, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=fc3fda29f660,38897,1731769023879, seqNum=-1, error=org.apache.hadoop.hbase.ipc.FailedServerException: Call to address=fc3fda29f660:38897 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: fc3fda29f660:38897 2024-11-16T14:58:11,829 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:meta,,1.1588230740, hostname=fc3fda29f660,38897,1731769023879, seqNum=-1 is org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: fc3fda29f660:38897 2024-11-16T14:58:11,829 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:meta,,1.1588230740, hostname=fc3fda29f660,38897,1731769023879, seqNum=-1 from cache 2024-11-16T14:58:12,026 WARN [BootstrapNodeManager {}] regionserver.BootstrapNodeManager(142): failed to get live region servers from master org.apache.hadoop.hbase.ipc.StoppedRpcClientException: Call to address=fc3fda29f660:43615 failed on local exception: org.apache.hadoop.hbase.ipc.StoppedRpcClientException at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:163) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.BootstrapNodeManager.getFromMaster(BootstrapNodeManager.java:140) ~[classes/:?] at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:539) ~[?:?] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:304) ~[?:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] at --------Future.get--------(Unknown Source) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:237) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:395) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:451) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$RpcChannelImplementation.callMethod(AbstractRpcClient.java:628) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$Stub.getLiveRegionServers(RegionServerStatusProtos.java:17191) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncClusterConnectionImpl.getLiveRegionServers(AsyncClusterConnectionImpl.java:139) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.BootstrapNodeManager.getFromMaster(BootstrapNodeManager.java:140) ~[classes/:?] at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:539) ~[?:?] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:304) ~[?:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.StoppedRpcClientException at org.apache.hadoop.hbase.ipc.AbstractRpcClient.getConnection(AbstractRpcClient.java:366) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:448) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 10 more 2024-11-16T14:58:12,139 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-16T14:58:12,139 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fc3fda29f660,38897,1731769023879, seqNum=-1] 2024-11-16T14:58:12,139 DEBUG [Async-Client-Retry-Timer-pool-0 {}] ipc.AbstractRpcClient(357): Not trying to connect to fc3fda29f660:38897 this server is in the failed servers list 2024-11-16T14:58:12,139 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=fc3fda29f660,38897,1731769023879, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=fc3fda29f660,38897,1731769023879, seqNum=-1, error=org.apache.hadoop.hbase.ipc.FailedServerException: Call to address=fc3fda29f660:38897 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: fc3fda29f660:38897 2024-11-16T14:58:12,139 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:meta,,1.1588230740, hostname=fc3fda29f660,38897,1731769023879, seqNum=-1 is org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: fc3fda29f660:38897 2024-11-16T14:58:12,140 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:meta,,1.1588230740, hostname=fc3fda29f660,38897,1731769023879, seqNum=-1 from cache 2024-11-16T14:58:12,648 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-16T14:58:12,649 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fc3fda29f660,38897,1731769023879, seqNum=-1] 2024-11-16T14:58:12,649 DEBUG [Async-Client-Retry-Timer-pool-0 {}] ipc.AbstractRpcClient(357): Not trying to connect to fc3fda29f660:38897 this server is in the failed servers list 2024-11-16T14:58:12,650 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=fc3fda29f660,38897,1731769023879, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=fc3fda29f660,38897,1731769023879, seqNum=-1, error=org.apache.hadoop.hbase.ipc.FailedServerException: Call to address=fc3fda29f660:38897 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: fc3fda29f660:38897 2024-11-16T14:58:12,650 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:meta,,1.1588230740, hostname=fc3fda29f660,38897,1731769023879, seqNum=-1 is org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: fc3fda29f660:38897 2024-11-16T14:58:12,650 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:meta,,1.1588230740, hostname=fc3fda29f660,38897,1731769023879, seqNum=-1 from cache 2024-11-16T14:58:13,659 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-16T14:58:13,659 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fc3fda29f660,38897,1731769023879, seqNum=-1] 2024-11-16T14:58:13,661 WARN [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.NettyRpcConnection$2(409): Exception encountered while connecting to the server fc3fda29f660:38897 org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: fc3fda29f660/172.17.0.2:38897 Caused by: java.net.ConnectException: finishConnect(..) failed: Connection refused at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.newConnectException0(Errors.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.handleConnectErrno(Errors.java:131) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Socket.finishConnect(Socket.java:359) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.doFinishConnect(AbstractEpollChannel.java:715) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:692) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:58:13,663 WARN [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRpcRetryingCaller(177): Call to fc3fda29f660,38897,1731769023879 for '' in 1588230740 of hbase:meta failed, tries = 6, maxAttempts = 46, timeout = 60000 ms, time elapsed = 2156 ms java.net.ConnectException: Call to address=fc3fda29f660:38897 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: fc3fda29f660/172.17.0.2:38897 at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:220) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:395) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.BufferCallBeforeInitHandler.userEventTriggered(BufferCallBeforeInitHandler.java:106) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:398) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:376) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireUserEventTriggered(AbstractChannelHandlerContext.java:368) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.userEventTriggered(DefaultChannelPipeline.java:1375) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:396) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:376) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireUserEventTriggered(DefaultChannelPipeline.java:862) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcConnection.failInit(NettyRpcConnection.java:210) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcConnection$2.fail(NettyRpcConnection.java:414) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcConnection$2.operationComplete(NettyRpcConnection.java:421) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcConnection$2.operationComplete(NettyRpcConnection.java:389) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:629) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:118) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.fulfillConnectPromise(AbstractEpollChannel.java:679) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:698) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: fc3fda29f660/172.17.0.2:38897 Caused by: java.net.ConnectException: finishConnect(..) failed: Connection refused at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.newConnectException0(Errors.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.handleConnectErrno(Errors.java:131) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Socket.finishConnect(Socket.java:359) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.doFinishConnect(AbstractEpollChannel.java:715) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:692) ~[hbase-shaded-netty-4.1.9.jar:?] ... 7 more 2024-11-16T14:58:13,665 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=fc3fda29f660,38897,1731769023879, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=fc3fda29f660,38897,1731769023879, seqNum=-1, error=java.net.ConnectException: Call to address=fc3fda29f660:38897 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: fc3fda29f660/172.17.0.2:38897 2024-11-16T14:58:13,665 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:meta,,1.1588230740, hostname=fc3fda29f660,38897,1731769023879, seqNum=-1 is java.net.ConnectException: finishConnect(..) failed: Connection refused 2024-11-16T14:58:13,665 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:meta,,1.1588230740, hostname=fc3fda29f660,38897,1731769023879, seqNum=-1 from cache 2024-11-16T14:58:13,666 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.FailedServers(52): Added failed server with address fc3fda29f660:38897 to list caused by org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: fc3fda29f660/172.17.0.2:38897 2024-11-16T14:58:15,679 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-16T14:58:15,679 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fc3fda29f660,38897,1731769023879, seqNum=-1] 2024-11-16T14:58:15,681 WARN [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.NettyRpcConnection$2(409): Exception encountered while connecting to the server fc3fda29f660:38897 org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: fc3fda29f660/172.17.0.2:38897 Caused by: java.net.ConnectException: finishConnect(..) failed: Connection refused at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.newConnectException0(Errors.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.handleConnectErrno(Errors.java:131) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Socket.finishConnect(Socket.java:359) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.doFinishConnect(AbstractEpollChannel.java:715) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:692) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:58:15,682 WARN [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRpcRetryingCaller(177): Call to fc3fda29f660,38897,1731769023879 for '' in 1588230740 of hbase:meta failed, tries = 7, maxAttempts = 46, timeout = 60000 ms, time elapsed = 4175 ms java.net.ConnectException: Call to address=fc3fda29f660:38897 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: fc3fda29f660/172.17.0.2:38897 at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:220) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:395) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.BufferCallBeforeInitHandler.userEventTriggered(BufferCallBeforeInitHandler.java:106) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:398) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:376) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireUserEventTriggered(AbstractChannelHandlerContext.java:368) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.userEventTriggered(DefaultChannelPipeline.java:1375) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:396) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:376) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireUserEventTriggered(DefaultChannelPipeline.java:862) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcConnection.failInit(NettyRpcConnection.java:210) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcConnection$2.fail(NettyRpcConnection.java:414) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcConnection$2.operationComplete(NettyRpcConnection.java:421) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcConnection$2.operationComplete(NettyRpcConnection.java:389) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:629) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:118) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.fulfillConnectPromise(AbstractEpollChannel.java:679) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:698) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: fc3fda29f660/172.17.0.2:38897 Caused by: java.net.ConnectException: finishConnect(..) failed: Connection refused at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.newConnectException0(Errors.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.handleConnectErrno(Errors.java:131) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Socket.finishConnect(Socket.java:359) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.doFinishConnect(AbstractEpollChannel.java:715) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:692) ~[hbase-shaded-netty-4.1.9.jar:?] ... 7 more 2024-11-16T14:58:15,683 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=fc3fda29f660,38897,1731769023879, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=fc3fda29f660,38897,1731769023879, seqNum=-1, error=java.net.ConnectException: Call to address=fc3fda29f660:38897 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: fc3fda29f660/172.17.0.2:38897 2024-11-16T14:58:15,684 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:meta,,1.1588230740, hostname=fc3fda29f660,38897,1731769023879, seqNum=-1 is java.net.ConnectException: finishConnect(..) failed: Connection refused 2024-11-16T14:58:15,684 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:meta,,1.1588230740, hostname=fc3fda29f660,38897,1731769023879, seqNum=-1 from cache 2024-11-16T14:58:15,685 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.FailedServers(52): Added failed server with address fc3fda29f660:38897 to list caused by org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: fc3fda29f660/172.17.0.2:38897 2024-11-16T14:58:19,689 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-16T14:58:19,690 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fc3fda29f660,38897,1731769023879, seqNum=-1] 2024-11-16T14:58:19,692 WARN [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.NettyRpcConnection$2(409): Exception encountered while connecting to the server fc3fda29f660:38897 org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: fc3fda29f660/172.17.0.2:38897 Caused by: java.net.ConnectException: finishConnect(..) failed: Connection refused at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.newConnectException0(Errors.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.handleConnectErrno(Errors.java:131) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Socket.finishConnect(Socket.java:359) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.doFinishConnect(AbstractEpollChannel.java:715) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:692) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:58:19,693 WARN [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRpcRetryingCaller(177): Call to fc3fda29f660,38897,1731769023879 for '' in 1588230740 of hbase:meta failed, tries = 8, maxAttempts = 46, timeout = 60000 ms, time elapsed = 8186 ms java.net.ConnectException: Call to address=fc3fda29f660:38897 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: fc3fda29f660/172.17.0.2:38897 at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:220) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:395) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.BufferCallBeforeInitHandler.userEventTriggered(BufferCallBeforeInitHandler.java:106) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:398) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:376) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireUserEventTriggered(AbstractChannelHandlerContext.java:368) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.userEventTriggered(DefaultChannelPipeline.java:1375) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:396) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:376) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireUserEventTriggered(DefaultChannelPipeline.java:862) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcConnection.failInit(NettyRpcConnection.java:210) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcConnection$2.fail(NettyRpcConnection.java:414) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcConnection$2.operationComplete(NettyRpcConnection.java:421) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcConnection$2.operationComplete(NettyRpcConnection.java:389) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:629) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:118) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.fulfillConnectPromise(AbstractEpollChannel.java:679) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:698) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: fc3fda29f660/172.17.0.2:38897 Caused by: java.net.ConnectException: finishConnect(..) failed: Connection refused at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.newConnectException0(Errors.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.handleConnectErrno(Errors.java:131) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Socket.finishConnect(Socket.java:359) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.doFinishConnect(AbstractEpollChannel.java:715) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:692) ~[hbase-shaded-netty-4.1.9.jar:?] ... 7 more 2024-11-16T14:58:19,694 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=fc3fda29f660,38897,1731769023879, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=fc3fda29f660,38897,1731769023879, seqNum=-1, error=java.net.ConnectException: Call to address=fc3fda29f660:38897 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: fc3fda29f660/172.17.0.2:38897 2024-11-16T14:58:19,694 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:meta,,1.1588230740, hostname=fc3fda29f660,38897,1731769023879, seqNum=-1 is java.net.ConnectException: finishConnect(..) failed: Connection refused 2024-11-16T14:58:19,694 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:meta,,1.1588230740, hostname=fc3fda29f660,38897,1731769023879, seqNum=-1 from cache 2024-11-16T14:58:19,695 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.FailedServers(52): Added failed server with address fc3fda29f660:38897 to list caused by org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: fc3fda29f660/172.17.0.2:38897 2024-11-16T14:58:29,739 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-16T14:58:29,739 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fc3fda29f660,38897,1731769023879, seqNum=-1] 2024-11-16T14:58:29,742 WARN [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.NettyRpcConnection$2(409): Exception encountered while connecting to the server fc3fda29f660:38897 org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: fc3fda29f660/172.17.0.2:38897 Caused by: java.net.ConnectException: finishConnect(..) failed: Connection refused at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.newConnectException0(Errors.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.handleConnectErrno(Errors.java:131) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Socket.finishConnect(Socket.java:359) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.doFinishConnect(AbstractEpollChannel.java:715) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:692) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:58:29,743 WARN [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRpcRetryingCaller(177): Call to fc3fda29f660,38897,1731769023879 for '' in 1588230740 of hbase:meta failed, tries = 9, maxAttempts = 46, timeout = 60000 ms, time elapsed = 18236 ms java.net.ConnectException: Call to address=fc3fda29f660:38897 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: fc3fda29f660/172.17.0.2:38897 at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:220) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:395) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.BufferCallBeforeInitHandler.userEventTriggered(BufferCallBeforeInitHandler.java:106) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:398) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:376) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireUserEventTriggered(AbstractChannelHandlerContext.java:368) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.userEventTriggered(DefaultChannelPipeline.java:1375) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:396) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:376) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireUserEventTriggered(DefaultChannelPipeline.java:862) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcConnection.failInit(NettyRpcConnection.java:210) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcConnection$2.fail(NettyRpcConnection.java:414) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcConnection$2.operationComplete(NettyRpcConnection.java:421) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcConnection$2.operationComplete(NettyRpcConnection.java:389) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:629) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:118) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.fulfillConnectPromise(AbstractEpollChannel.java:679) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:698) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: fc3fda29f660/172.17.0.2:38897 Caused by: java.net.ConnectException: finishConnect(..) failed: Connection refused at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.newConnectException0(Errors.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.handleConnectErrno(Errors.java:131) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Socket.finishConnect(Socket.java:359) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.doFinishConnect(AbstractEpollChannel.java:715) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:692) ~[hbase-shaded-netty-4.1.9.jar:?] ... 7 more 2024-11-16T14:58:29,744 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=fc3fda29f660,38897,1731769023879, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=fc3fda29f660,38897,1731769023879, seqNum=-1, error=java.net.ConnectException: Call to address=fc3fda29f660:38897 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: fc3fda29f660/172.17.0.2:38897 2024-11-16T14:58:29,744 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:meta,,1.1588230740, hostname=fc3fda29f660,38897,1731769023879, seqNum=-1 is java.net.ConnectException: finishConnect(..) failed: Connection refused 2024-11-16T14:58:29,745 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:meta,,1.1588230740, hostname=fc3fda29f660,38897,1731769023879, seqNum=-1 from cache 2024-11-16T14:58:29,745 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.FailedServers(52): Added failed server with address fc3fda29f660:38897 to list caused by org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: fc3fda29f660/172.17.0.2:38897 2024-11-16T14:58:32,165 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-16T14:58:39,809 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-16T14:58:39,809 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fc3fda29f660,38897,1731769023879, seqNum=-1] 2024-11-16T14:58:39,812 WARN [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.NettyRpcConnection$2(409): Exception encountered while connecting to the server fc3fda29f660:38897 org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: fc3fda29f660/172.17.0.2:38897 Caused by: java.net.ConnectException: finishConnect(..) failed: Connection refused at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.newConnectException0(Errors.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.handleConnectErrno(Errors.java:131) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Socket.finishConnect(Socket.java:359) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.doFinishConnect(AbstractEpollChannel.java:715) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:692) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:58:39,813 WARN [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRpcRetryingCaller(177): Call to fc3fda29f660,38897,1731769023879 for '' in 1588230740 of hbase:meta failed, tries = 10, maxAttempts = 46, timeout = 60000 ms, time elapsed = 28306 ms java.net.ConnectException: Call to address=fc3fda29f660:38897 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: fc3fda29f660/172.17.0.2:38897 at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:220) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:395) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.BufferCallBeforeInitHandler.userEventTriggered(BufferCallBeforeInitHandler.java:106) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:398) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:376) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireUserEventTriggered(AbstractChannelHandlerContext.java:368) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.userEventTriggered(DefaultChannelPipeline.java:1375) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:396) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:376) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireUserEventTriggered(DefaultChannelPipeline.java:862) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcConnection.failInit(NettyRpcConnection.java:210) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcConnection$2.fail(NettyRpcConnection.java:414) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcConnection$2.operationComplete(NettyRpcConnection.java:421) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcConnection$2.operationComplete(NettyRpcConnection.java:389) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:629) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:118) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.fulfillConnectPromise(AbstractEpollChannel.java:679) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:698) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: fc3fda29f660/172.17.0.2:38897 Caused by: java.net.ConnectException: finishConnect(..) failed: Connection refused at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.newConnectException0(Errors.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.handleConnectErrno(Errors.java:131) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Socket.finishConnect(Socket.java:359) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.doFinishConnect(AbstractEpollChannel.java:715) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:692) ~[hbase-shaded-netty-4.1.9.jar:?] ... 7 more 2024-11-16T14:58:39,814 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=fc3fda29f660,38897,1731769023879, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=fc3fda29f660,38897,1731769023879, seqNum=-1, error=java.net.ConnectException: Call to address=fc3fda29f660:38897 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: fc3fda29f660/172.17.0.2:38897 2024-11-16T14:58:39,815 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:meta,,1.1588230740, hostname=fc3fda29f660,38897,1731769023879, seqNum=-1 is java.net.ConnectException: finishConnect(..) failed: Connection refused 2024-11-16T14:58:39,815 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:meta,,1.1588230740, hostname=fc3fda29f660,38897,1731769023879, seqNum=-1 from cache 2024-11-16T14:58:39,815 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.FailedServers(52): Added failed server with address fc3fda29f660:38897 to list caused by org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: fc3fda29f660/172.17.0.2:38897 2024-11-16T14:58:49,919 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-16T14:58:49,919 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fc3fda29f660,38897,1731769023879, seqNum=-1] 2024-11-16T14:58:49,921 WARN [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.NettyRpcConnection$2(409): Exception encountered while connecting to the server fc3fda29f660:38897 org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: fc3fda29f660/172.17.0.2:38897 Caused by: java.net.ConnectException: finishConnect(..) failed: Connection refused at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.newConnectException0(Errors.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.handleConnectErrno(Errors.java:131) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Socket.finishConnect(Socket.java:359) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.doFinishConnect(AbstractEpollChannel.java:715) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:692) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:58:49,922 WARN [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRpcRetryingCaller(177): Call to fc3fda29f660,38897,1731769023879 for '' in 1588230740 of hbase:meta failed, tries = 11, maxAttempts = 46, timeout = 60000 ms, time elapsed = 38415 ms java.net.ConnectException: Call to address=fc3fda29f660:38897 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: fc3fda29f660/172.17.0.2:38897 at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:220) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:395) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.BufferCallBeforeInitHandler.userEventTriggered(BufferCallBeforeInitHandler.java:106) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:398) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:376) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireUserEventTriggered(AbstractChannelHandlerContext.java:368) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.userEventTriggered(DefaultChannelPipeline.java:1375) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:396) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:376) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireUserEventTriggered(DefaultChannelPipeline.java:862) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcConnection.failInit(NettyRpcConnection.java:210) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcConnection$2.fail(NettyRpcConnection.java:414) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcConnection$2.operationComplete(NettyRpcConnection.java:421) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcConnection$2.operationComplete(NettyRpcConnection.java:389) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:629) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:118) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.fulfillConnectPromise(AbstractEpollChannel.java:679) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:698) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: fc3fda29f660/172.17.0.2:38897 Caused by: java.net.ConnectException: finishConnect(..) failed: Connection refused at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.newConnectException0(Errors.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.handleConnectErrno(Errors.java:131) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Socket.finishConnect(Socket.java:359) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.doFinishConnect(AbstractEpollChannel.java:715) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:692) ~[hbase-shaded-netty-4.1.9.jar:?] ... 7 more 2024-11-16T14:58:49,923 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=fc3fda29f660,38897,1731769023879, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=fc3fda29f660,38897,1731769023879, seqNum=-1, error=java.net.ConnectException: Call to address=fc3fda29f660:38897 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: fc3fda29f660/172.17.0.2:38897 2024-11-16T14:58:49,923 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:meta,,1.1588230740, hostname=fc3fda29f660,38897,1731769023879, seqNum=-1 is java.net.ConnectException: finishConnect(..) failed: Connection refused 2024-11-16T14:58:49,924 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:meta,,1.1588230740, hostname=fc3fda29f660,38897,1731769023879, seqNum=-1 from cache 2024-11-16T14:58:49,924 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.FailedServers(52): Added failed server with address fc3fda29f660:38897 to list caused by org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: fc3fda29f660/172.17.0.2:38897 2024-11-16T14:58:59,989 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-16T14:58:59,989 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fc3fda29f660,38897,1731769023879, seqNum=-1] 2024-11-16T14:58:59,991 WARN [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.NettyRpcConnection$2(409): Exception encountered while connecting to the server fc3fda29f660:38897 org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: fc3fda29f660/172.17.0.2:38897 Caused by: java.net.ConnectException: finishConnect(..) failed: Connection refused at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.newConnectException0(Errors.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.handleConnectErrno(Errors.java:131) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Socket.finishConnect(Socket.java:359) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.doFinishConnect(AbstractEpollChannel.java:715) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:692) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:58:59,993 WARN [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRpcRetryingCaller(177): Call to fc3fda29f660,38897,1731769023879 for '' in 1588230740 of hbase:meta failed, tries = 12, maxAttempts = 46, timeout = 60000 ms, time elapsed = 48485 ms java.net.ConnectException: Call to address=fc3fda29f660:38897 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: fc3fda29f660/172.17.0.2:38897 at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:220) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:395) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.BufferCallBeforeInitHandler.userEventTriggered(BufferCallBeforeInitHandler.java:106) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:398) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:376) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireUserEventTriggered(AbstractChannelHandlerContext.java:368) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.userEventTriggered(DefaultChannelPipeline.java:1375) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:396) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:376) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireUserEventTriggered(DefaultChannelPipeline.java:862) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcConnection.failInit(NettyRpcConnection.java:210) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcConnection$2.fail(NettyRpcConnection.java:414) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcConnection$2.operationComplete(NettyRpcConnection.java:421) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcConnection$2.operationComplete(NettyRpcConnection.java:389) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:629) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:118) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.fulfillConnectPromise(AbstractEpollChannel.java:679) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:698) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: fc3fda29f660/172.17.0.2:38897 Caused by: java.net.ConnectException: finishConnect(..) failed: Connection refused at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.newConnectException0(Errors.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.handleConnectErrno(Errors.java:131) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Socket.finishConnect(Socket.java:359) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.doFinishConnect(AbstractEpollChannel.java:715) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:692) ~[hbase-shaded-netty-4.1.9.jar:?] ... 7 more 2024-11-16T14:58:59,994 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=fc3fda29f660,38897,1731769023879, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=fc3fda29f660,38897,1731769023879, seqNum=-1, error=java.net.ConnectException: Call to address=fc3fda29f660:38897 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: fc3fda29f660/172.17.0.2:38897 2024-11-16T14:58:59,994 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:meta,,1.1588230740, hostname=fc3fda29f660,38897,1731769023879, seqNum=-1 is java.net.ConnectException: finishConnect(..) failed: Connection refused 2024-11-16T14:58:59,994 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:meta,,1.1588230740, hostname=fc3fda29f660,38897,1731769023879, seqNum=-1 from cache 2024-11-16T14:58:59,995 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.FailedServers(52): Added failed server with address fc3fda29f660:38897 to list caused by org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: fc3fda29f660/172.17.0.2:38897 2024-11-16T14:59:02,165 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-16T14:59:05,279 WARN [regionserver/fc3fda29f660:0.Chore.1 {}] hbase.ExecutorStatusChore(69): RS_COMPACTED_FILES_DISCHARGER's size info, queued: 21, running: 1 2024-11-16T14:59:09,979 INFO [regionserver/fc3fda29f660:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: CompactedHFilesCleaner was stopped 2024-11-16T14:59:11,510 WARN [Time-limited test {}] procedure.MasterProcedureTestingUtility$3(130): Failed to load meta org.apache.hadoop.hbase.client.RetriesExhaustedException: Failed after attempts=13, exceptions: 2024-11-16T14:58:11.509Z, java.net.ConnectException: Call to address=fc3fda29f660:38897 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: fc3fda29f660/172.17.0.2:38897 2024-11-16T14:58:11.620Z, org.apache.hadoop.hbase.ipc.FailedServerException: Call to address=fc3fda29f660:38897 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: fc3fda29f660:38897 2024-11-16T14:58:11.829Z, org.apache.hadoop.hbase.ipc.FailedServerException: Call to address=fc3fda29f660:38897 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: fc3fda29f660:38897 2024-11-16T14:58:12.140Z, org.apache.hadoop.hbase.ipc.FailedServerException: Call to address=fc3fda29f660:38897 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: fc3fda29f660:38897 2024-11-16T14:58:12.650Z, org.apache.hadoop.hbase.ipc.FailedServerException: Call to address=fc3fda29f660:38897 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: fc3fda29f660:38897 2024-11-16T14:58:13.665Z, java.net.ConnectException: Call to address=fc3fda29f660:38897 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: fc3fda29f660/172.17.0.2:38897 2024-11-16T14:58:15.684Z, java.net.ConnectException: Call to address=fc3fda29f660:38897 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: fc3fda29f660/172.17.0.2:38897 2024-11-16T14:58:19.694Z, java.net.ConnectException: Call to address=fc3fda29f660:38897 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: fc3fda29f660/172.17.0.2:38897 2024-11-16T14:58:29.745Z, java.net.ConnectException: Call to address=fc3fda29f660:38897 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: fc3fda29f660/172.17.0.2:38897 2024-11-16T14:58:39.815Z, java.net.ConnectException: Call to address=fc3fda29f660:38897 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: fc3fda29f660/172.17.0.2:38897 2024-11-16T14:58:49.924Z, java.net.ConnectException: Call to address=fc3fda29f660:38897 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: fc3fda29f660/172.17.0.2:38897 2024-11-16T14:58:59.994Z, java.net.ConnectException: Call to address=fc3fda29f660:38897 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: fc3fda29f660/172.17.0.2:38897 at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:163) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncTableResultScanner.next(AsyncTableResultScanner.java:173) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.MetaTableAccessor.scanMeta(MetaTableAccessor.java:493) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.MetaTableAccessor.scanMeta(MetaTableAccessor.java:461) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.MetaTableAccessor.scanMeta(MetaTableAccessor.java:422) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.MetaTableAccessor.fullScanRegions(MetaTableAccessor.java:108) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionStateStore.visitMeta(RegionStateStore.java:108) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.AssignmentManager.loadMeta(AssignmentManager.java:1907) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.AssignmentManager.joinCluster(AssignmentManager.java:1769) ~[classes/:?] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility$3.call(MasterProcedureTestingUtility.java:126) ~[test-classes/:?] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility$3.call(MasterProcedureTestingUtility.java:121) ~[test-classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:143) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT-tests.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT-tests.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.testRecoveryAndDoubleExecution(MasterProcedureTestingUtility.java:477) ~[test-classes/:?] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.testRecoveryAndDoubleExecution(MasterProcedureTestingUtility.java:486) ~[test-classes/:?] at org.apache.hadoop.hbase.master.procedure.TestSCPBase.testRecoveryAndDoubleExecution(TestSCPBase.java:114) ~[test-classes/:?] at org.apache.hadoop.hbase.master.procedure.TestSCPWithMeta.testRecoveryAndDoubleExecutionOnRsWithMeta(TestSCPWithMeta.java:36) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.completeExceptionally(AsyncRpcRetryingCaller.java:109) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.doCall(AsyncSingleRequestRpcRetryingCaller.java:102) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.lambda$tryScheduleRetry$1(AsyncRpcRetryingCaller.java:139) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelTimeout.run(HashedWheelTimer.java:713) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.ImmediateExecutor.execute(ImmediateExecutor.java:34) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelTimeout.expire(HashedWheelTimer.java:701) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelBucket.expireTimeouts(HashedWheelTimer.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:501) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.net.ConnectException: Call to address=fc3fda29f660:38897 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: fc3fda29f660/172.17.0.2:38897 at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:220) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:395) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.BufferCallBeforeInitHandler.userEventTriggered(BufferCallBeforeInitHandler.java:106) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:398) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:376) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireUserEventTriggered(AbstractChannelHandlerContext.java:368) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.userEventTriggered(DefaultChannelPipeline.java:1375) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:396) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:376) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireUserEventTriggered(DefaultChannelPipeline.java:862) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcConnection.failInit(NettyRpcConnection.java:210) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcConnection$2.fail(NettyRpcConnection.java:414) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcConnection$2.operationComplete(NettyRpcConnection.java:421) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcConnection$2.operationComplete(NettyRpcConnection.java:389) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:629) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:118) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.fulfillConnectPromise(AbstractEpollChannel.java:679) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:698) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] ... 1 more Caused by: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: fc3fda29f660/172.17.0.2:38897 Caused by: java.net.ConnectException: finishConnect(..) failed: Connection refused at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.newConnectException0(Errors.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.handleConnectErrno(Errors.java:131) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Socket.finishConnect(Socket.java:359) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.doFinishConnect(AbstractEpollChannel.java:715) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:692) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] ... 1 more 2024-11-16T14:59:11,513 INFO [Time-limited test {}] procedure.MasterProcedureTestingUtility(473): Restart 2 exec state=pid=57, state=RUNNABLE:SERVER_CRASH_ASSIGN_META, hasLock=true; ServerCrashProcedure fc3fda29f660,38897,1731769023879, splitWal=true, meta=true 2024-11-16T14:59:11,513 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-16T14:59:11,513 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-16T14:59:11,513 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-16T14:59:11,513 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-16T14:59:11,514 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-16T14:59:11,514 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-16T14:59:11,514 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-16T14:59:11,514 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-16T14:59:11,516 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 1 msec 2024-11-16T14:59:11,522 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=57, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-16T14:59:11,522 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=58, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-16T14:59:11,522 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=59, ppid=58, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-16T14:59:11,522 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-16T14:59:11,523 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true 2024-11-16T14:59:11,524 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=57, state=RUNNABLE:SERVER_CRASH_DELETE_SPLIT_META_WALS_DIR, hasLock=false; ServerCrashProcedure fc3fda29f660,38897,1731769023879, splitWal=true, meta=true 2024-11-16T14:59:11,524 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure fc3fda29f660%2C38897%2C1731769023879.meta.1731769025887.meta, worker=fc3fda29f660,42985,1731769023951 2024-11-16T14:59:11,524 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure fc3fda29f660%2C38897%2C1731769023879.meta.1731769025887.meta, worker=fc3fda29f660,42985,1731769023951 2024-11-16T14:59:11,524 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=57, state=RUNNABLE:SERVER_CRASH_DELETE_SPLIT_META_WALS_DIR, hasLock=false; ServerCrashProcedure fc3fda29f660,38897,1731769023879, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-16T14:59:11,524 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure fc3fda29f660%2C38897%2C1731769023879.meta.1731769025887.meta, worker=fc3fda29f660,42985,1731769023951 didn't hold the lock before restarting, skip acquiring lock. 2024-11-16T14:59:11,524 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure fc3fda29f660%2C38897%2C1731769023879.meta.1731769025887.meta, worker=fc3fda29f660,42985,1731769023951 didn't hold the lock before restarting, skip acquiring lock. 2024-11-16T14:59:11,524 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 7 msec 2024-11-16T14:59:11,525 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=fc3fda29f660,38897,1731769023879, regionLocation=fc3fda29f660,38897,1731769023879, openSeqNum=2 2024-11-16T14:59:11,525 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=fc3fda29f660,38897,1731769023879, table=hbase:meta, region=1588230740 2024-11-16T14:59:11,525 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as fc3fda29f660,38897,1731769023879, state=OPEN 2024-11-16T14:59:11,529 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=43615 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:630) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:59:11,529 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=43615 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:630) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:59:11,600 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39089-0x101456fc9c70001, quorum=127.0.0.1:63224, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-16T14:59:11,600 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43615-0x101456fc9c70000, quorum=127.0.0.1:63224, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-16T14:59:11,600 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42985-0x101456fc9c70003, quorum=127.0.0.1:63224, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-16T14:59:11,600 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-16T14:59:11,600 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-16T14:59:11,600 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-16T14:59:11,602 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731769181601 2024-11-16T14:59:11,602 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-16T14:59:11,602 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-16T14:59:11,602 DEBUG [Time-limited test {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fc3fda29f660,38897,1731769023879, seqNum=-1] 2024-11-16T14:59:11,604 WARN [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.NettyRpcConnection$2(409): Exception encountered while connecting to the server fc3fda29f660:38897 org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: fc3fda29f660/172.17.0.2:38897 Caused by: java.net.ConnectException: finishConnect(..) failed: Connection refused at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.newConnectException0(Errors.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.handleConnectErrno(Errors.java:131) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Socket.finishConnect(Socket.java:359) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.doFinishConnect(AbstractEpollChannel.java:715) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:692) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:59:11,604 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=fc3fda29f660,38897,1731769023879, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=fc3fda29f660,38897,1731769023879, seqNum=-1, error=java.net.ConnectException: Call to address=fc3fda29f660:38897 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: fc3fda29f660/172.17.0.2:38897 2024-11-16T14:59:11,604 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:meta,,1.1588230740, hostname=fc3fda29f660,38897,1731769023879, seqNum=-1 is java.net.ConnectException: finishConnect(..) failed: Connection refused 2024-11-16T14:59:11,604 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:meta,,1.1588230740, hostname=fc3fda29f660,38897,1731769023879, seqNum=-1 from cache 2024-11-16T14:59:11,605 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.FailedServers(52): Added failed server with address fc3fda29f660:38897 to list caused by org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: fc3fda29f660/172.17.0.2:38897 2024-11-16T14:59:11,605 INFO [PEWorker-1 {}] master.SplitLogManager(171): hdfs://localhost:42223/user/jenkins/test-data/a8176cb1-2ebb-b530-b3c7-53c9e53c186e/WALs/fc3fda29f660,38897,1731769023879-splitting dir is empty, no logs to split. 2024-11-16T14:59:11,605 INFO [PEWorker-1 {}] master.SplitWALManager(105): fc3fda29f660,38897,1731769023879 WAL count=0, meta=true 2024-11-16T14:59:11,606 DEBUG [PEWorker-1 {}] procedure.ServerCrashProcedure(329): Check if fc3fda29f660,38897,1731769023879 WAL splitting is done? wals=0, meta=true 2024-11-16T14:59:11,606 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(132): Toggle KILL before store update to: true 2024-11-16T14:59:11,608 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=60, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-11-16T14:59:11,608 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(132): Toggle KILL before store update to: false 2024-11-16T14:59:11,608 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update: pid=57, state=WAITING:SERVER_CRASH_GET_REGIONS, hasLock=true; ServerCrashProcedure fc3fda29f660,38897,1731769023879, splitWal=true, meta=true 2024-11-16T14:59:11,608 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-16T14:59:11,609 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update: pid=57, state=WAITING:SERVER_CRASH_GET_REGIONS, hasLock=true; ServerCrashProcedure fc3fda29f660,38897,1731769023879, splitWal=true, meta=true at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1879) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T14:59:11,708 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-16T14:59:11,708 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fc3fda29f660,38897,1731769023879, seqNum=-1] 2024-11-16T14:59:11,709 DEBUG [Async-Client-Retry-Timer-pool-0 {}] ipc.AbstractRpcClient(357): Not trying to connect to fc3fda29f660:38897 this server is in the failed servers list 2024-11-16T14:59:11,709 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=fc3fda29f660,38897,1731769023879, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=fc3fda29f660,38897,1731769023879, seqNum=-1, error=org.apache.hadoop.hbase.ipc.FailedServerException: Call to address=fc3fda29f660:38897 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: fc3fda29f660:38897 2024-11-16T14:59:11,710 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:meta,,1.1588230740, hostname=fc3fda29f660,38897,1731769023879, seqNum=-1 is org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: fc3fda29f660:38897 2024-11-16T14:59:11,710 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:meta,,1.1588230740, hostname=fc3fda29f660,38897,1731769023879, seqNum=-1 from cache 2024-11-16T14:59:11,919 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-16T14:59:11,919 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fc3fda29f660,38897,1731769023879, seqNum=-1] 2024-11-16T14:59:11,919 DEBUG [Async-Client-Retry-Timer-pool-0 {}] ipc.AbstractRpcClient(357): Not trying to connect to fc3fda29f660:38897 this server is in the failed servers list 2024-11-16T14:59:11,920 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=fc3fda29f660,38897,1731769023879, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=fc3fda29f660,38897,1731769023879, seqNum=-1, error=org.apache.hadoop.hbase.ipc.FailedServerException: Call to address=fc3fda29f660:38897 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: fc3fda29f660:38897 2024-11-16T14:59:11,920 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:meta,,1.1588230740, hostname=fc3fda29f660,38897,1731769023879, seqNum=-1 is org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: fc3fda29f660:38897 2024-11-16T14:59:11,920 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:meta,,1.1588230740, hostname=fc3fda29f660,38897,1731769023879, seqNum=-1 from cache 2024-11-16T14:59:12,229 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-16T14:59:12,229 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fc3fda29f660,38897,1731769023879, seqNum=-1] 2024-11-16T14:59:12,230 DEBUG [Async-Client-Retry-Timer-pool-0 {}] ipc.AbstractRpcClient(357): Not trying to connect to fc3fda29f660:38897 this server is in the failed servers list 2024-11-16T14:59:12,230 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=fc3fda29f660,38897,1731769023879, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=fc3fda29f660,38897,1731769023879, seqNum=-1, error=org.apache.hadoop.hbase.ipc.FailedServerException: Call to address=fc3fda29f660:38897 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: fc3fda29f660:38897 2024-11-16T14:59:12,230 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:meta,,1.1588230740, hostname=fc3fda29f660,38897,1731769023879, seqNum=-1 is org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: fc3fda29f660:38897 2024-11-16T14:59:12,231 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:meta,,1.1588230740, hostname=fc3fda29f660,38897,1731769023879, seqNum=-1 from cache 2024-11-16T14:59:12,739 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-16T14:59:12,739 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fc3fda29f660,38897,1731769023879, seqNum=-1] 2024-11-16T14:59:12,740 DEBUG [Async-Client-Retry-Timer-pool-0 {}] ipc.AbstractRpcClient(357): Not trying to connect to fc3fda29f660:38897 this server is in the failed servers list 2024-11-16T14:59:12,740 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=fc3fda29f660,38897,1731769023879, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=fc3fda29f660,38897,1731769023879, seqNum=-1, error=org.apache.hadoop.hbase.ipc.FailedServerException: Call to address=fc3fda29f660:38897 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: fc3fda29f660:38897 2024-11-16T14:59:12,741 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:meta,,1.1588230740, hostname=fc3fda29f660,38897,1731769023879, seqNum=-1 is org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: fc3fda29f660:38897 2024-11-16T14:59:12,741 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:meta,,1.1588230740, hostname=fc3fda29f660,38897,1731769023879, seqNum=-1 from cache 2024-11-16T14:59:13,759 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-16T14:59:13,759 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fc3fda29f660,38897,1731769023879, seqNum=-1] 2024-11-16T14:59:13,761 WARN [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.NettyRpcConnection$2(409): Exception encountered while connecting to the server fc3fda29f660:38897 org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: fc3fda29f660/172.17.0.2:38897 Caused by: java.net.ConnectException: finishConnect(..) failed: Connection refused at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.newConnectException0(Errors.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.handleConnectErrno(Errors.java:131) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Socket.finishConnect(Socket.java:359) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.doFinishConnect(AbstractEpollChannel.java:715) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:692) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:59:13,762 WARN [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRpcRetryingCaller(177): Call to fc3fda29f660,38897,1731769023879 for '' in 1588230740 of hbase:meta failed, tries = 6, maxAttempts = 46, timeout = 60000 ms, time elapsed = 2159 ms java.net.ConnectException: Call to address=fc3fda29f660:38897 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: fc3fda29f660/172.17.0.2:38897 at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:220) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:395) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.BufferCallBeforeInitHandler.userEventTriggered(BufferCallBeforeInitHandler.java:106) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:398) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:376) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireUserEventTriggered(AbstractChannelHandlerContext.java:368) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.userEventTriggered(DefaultChannelPipeline.java:1375) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:396) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:376) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireUserEventTriggered(DefaultChannelPipeline.java:862) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcConnection.failInit(NettyRpcConnection.java:210) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcConnection$2.fail(NettyRpcConnection.java:414) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcConnection$2.operationComplete(NettyRpcConnection.java:421) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcConnection$2.operationComplete(NettyRpcConnection.java:389) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:629) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:118) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.fulfillConnectPromise(AbstractEpollChannel.java:679) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:698) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: fc3fda29f660/172.17.0.2:38897 Caused by: java.net.ConnectException: finishConnect(..) failed: Connection refused at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.newConnectException0(Errors.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.handleConnectErrno(Errors.java:131) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Socket.finishConnect(Socket.java:359) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.doFinishConnect(AbstractEpollChannel.java:715) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:692) ~[hbase-shaded-netty-4.1.9.jar:?] ... 7 more 2024-11-16T14:59:13,763 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=fc3fda29f660,38897,1731769023879, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=fc3fda29f660,38897,1731769023879, seqNum=-1, error=java.net.ConnectException: Call to address=fc3fda29f660:38897 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: fc3fda29f660/172.17.0.2:38897 2024-11-16T14:59:13,763 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:meta,,1.1588230740, hostname=fc3fda29f660,38897,1731769023879, seqNum=-1 is java.net.ConnectException: finishConnect(..) failed: Connection refused 2024-11-16T14:59:13,763 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:meta,,1.1588230740, hostname=fc3fda29f660,38897,1731769023879, seqNum=-1 from cache 2024-11-16T14:59:13,764 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.FailedServers(52): Added failed server with address fc3fda29f660:38897 to list caused by org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: fc3fda29f660/172.17.0.2:38897 2024-11-16T14:59:15,769 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-16T14:59:15,769 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fc3fda29f660,38897,1731769023879, seqNum=-1] 2024-11-16T14:59:15,771 WARN [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.NettyRpcConnection$2(409): Exception encountered while connecting to the server fc3fda29f660:38897 org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: fc3fda29f660/172.17.0.2:38897 Caused by: java.net.ConnectException: finishConnect(..) failed: Connection refused at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.newConnectException0(Errors.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.handleConnectErrno(Errors.java:131) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Socket.finishConnect(Socket.java:359) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.doFinishConnect(AbstractEpollChannel.java:715) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:692) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:59:15,773 WARN [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRpcRetryingCaller(177): Call to fc3fda29f660,38897,1731769023879 for '' in 1588230740 of hbase:meta failed, tries = 7, maxAttempts = 46, timeout = 60000 ms, time elapsed = 4170 ms java.net.ConnectException: Call to address=fc3fda29f660:38897 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: fc3fda29f660/172.17.0.2:38897 at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:220) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:395) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.BufferCallBeforeInitHandler.userEventTriggered(BufferCallBeforeInitHandler.java:106) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:398) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:376) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireUserEventTriggered(AbstractChannelHandlerContext.java:368) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.userEventTriggered(DefaultChannelPipeline.java:1375) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:396) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:376) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireUserEventTriggered(DefaultChannelPipeline.java:862) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcConnection.failInit(NettyRpcConnection.java:210) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcConnection$2.fail(NettyRpcConnection.java:414) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcConnection$2.operationComplete(NettyRpcConnection.java:421) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcConnection$2.operationComplete(NettyRpcConnection.java:389) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:629) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:118) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.fulfillConnectPromise(AbstractEpollChannel.java:679) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:698) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: fc3fda29f660/172.17.0.2:38897 Caused by: java.net.ConnectException: finishConnect(..) failed: Connection refused at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.newConnectException0(Errors.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.handleConnectErrno(Errors.java:131) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Socket.finishConnect(Socket.java:359) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.doFinishConnect(AbstractEpollChannel.java:715) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:692) ~[hbase-shaded-netty-4.1.9.jar:?] ... 7 more 2024-11-16T14:59:15,774 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=fc3fda29f660,38897,1731769023879, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=fc3fda29f660,38897,1731769023879, seqNum=-1, error=java.net.ConnectException: Call to address=fc3fda29f660:38897 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: fc3fda29f660/172.17.0.2:38897 2024-11-16T14:59:15,774 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:meta,,1.1588230740, hostname=fc3fda29f660,38897,1731769023879, seqNum=-1 is java.net.ConnectException: finishConnect(..) failed: Connection refused 2024-11-16T14:59:15,774 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:meta,,1.1588230740, hostname=fc3fda29f660,38897,1731769023879, seqNum=-1 from cache 2024-11-16T14:59:15,775 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.FailedServers(52): Added failed server with address fc3fda29f660:38897 to list caused by org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: fc3fda29f660/172.17.0.2:38897 2024-11-16T14:59:18,028 WARN [BootstrapNodeManager {}] regionserver.BootstrapNodeManager(142): failed to get live region servers from master org.apache.hadoop.hbase.ipc.StoppedRpcClientException: Call to address=fc3fda29f660:43615 failed on local exception: org.apache.hadoop.hbase.ipc.StoppedRpcClientException at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:163) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.BootstrapNodeManager.getFromMaster(BootstrapNodeManager.java:140) ~[classes/:?] at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:539) ~[?:?] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:304) ~[?:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] at --------Future.get--------(Unknown Source) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:237) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:395) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:451) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$RpcChannelImplementation.callMethod(AbstractRpcClient.java:628) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$Stub.getLiveRegionServers(RegionServerStatusProtos.java:17191) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncClusterConnectionImpl.getLiveRegionServers(AsyncClusterConnectionImpl.java:139) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.BootstrapNodeManager.getFromMaster(BootstrapNodeManager.java:140) ~[classes/:?] at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:539) ~[?:?] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:304) ~[?:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.StoppedRpcClientException at org.apache.hadoop.hbase.ipc.AbstractRpcClient.getConnection(AbstractRpcClient.java:366) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:448) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 10 more 2024-11-16T14:59:19,808 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-16T14:59:19,809 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fc3fda29f660,38897,1731769023879, seqNum=-1] 2024-11-16T14:59:19,810 WARN [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.NettyRpcConnection$2(409): Exception encountered while connecting to the server fc3fda29f660:38897 org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: fc3fda29f660/172.17.0.2:38897 Caused by: java.net.ConnectException: finishConnect(..) failed: Connection refused at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.newConnectException0(Errors.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.handleConnectErrno(Errors.java:131) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Socket.finishConnect(Socket.java:359) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.doFinishConnect(AbstractEpollChannel.java:715) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:692) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:59:19,811 WARN [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRpcRetryingCaller(177): Call to fc3fda29f660,38897,1731769023879 for '' in 1588230740 of hbase:meta failed, tries = 8, maxAttempts = 46, timeout = 60000 ms, time elapsed = 8208 ms java.net.ConnectException: Call to address=fc3fda29f660:38897 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: fc3fda29f660/172.17.0.2:38897 at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:220) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:395) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.BufferCallBeforeInitHandler.userEventTriggered(BufferCallBeforeInitHandler.java:106) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:398) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:376) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireUserEventTriggered(AbstractChannelHandlerContext.java:368) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.userEventTriggered(DefaultChannelPipeline.java:1375) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:396) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:376) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireUserEventTriggered(DefaultChannelPipeline.java:862) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcConnection.failInit(NettyRpcConnection.java:210) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcConnection$2.fail(NettyRpcConnection.java:414) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcConnection$2.operationComplete(NettyRpcConnection.java:421) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcConnection$2.operationComplete(NettyRpcConnection.java:389) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:629) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:118) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.fulfillConnectPromise(AbstractEpollChannel.java:679) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:698) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: fc3fda29f660/172.17.0.2:38897 Caused by: java.net.ConnectException: finishConnect(..) failed: Connection refused at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.newConnectException0(Errors.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.handleConnectErrno(Errors.java:131) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Socket.finishConnect(Socket.java:359) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.doFinishConnect(AbstractEpollChannel.java:715) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:692) ~[hbase-shaded-netty-4.1.9.jar:?] ... 7 more 2024-11-16T14:59:19,812 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=fc3fda29f660,38897,1731769023879, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=fc3fda29f660,38897,1731769023879, seqNum=-1, error=java.net.ConnectException: Call to address=fc3fda29f660:38897 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: fc3fda29f660/172.17.0.2:38897 2024-11-16T14:59:19,812 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:meta,,1.1588230740, hostname=fc3fda29f660,38897,1731769023879, seqNum=-1 is java.net.ConnectException: finishConnect(..) failed: Connection refused 2024-11-16T14:59:19,812 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:meta,,1.1588230740, hostname=fc3fda29f660,38897,1731769023879, seqNum=-1 from cache 2024-11-16T14:59:19,812 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.FailedServers(52): Added failed server with address fc3fda29f660:38897 to list caused by org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: fc3fda29f660/172.17.0.2:38897 2024-11-16T14:59:29,839 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-16T14:59:29,839 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fc3fda29f660,38897,1731769023879, seqNum=-1] 2024-11-16T14:59:29,841 WARN [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.NettyRpcConnection$2(409): Exception encountered while connecting to the server fc3fda29f660:38897 org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: fc3fda29f660/172.17.0.2:38897 Caused by: java.net.ConnectException: finishConnect(..) failed: Connection refused at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.newConnectException0(Errors.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.handleConnectErrno(Errors.java:131) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Socket.finishConnect(Socket.java:359) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.doFinishConnect(AbstractEpollChannel.java:715) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:692) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:59:29,843 WARN [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRpcRetryingCaller(177): Call to fc3fda29f660,38897,1731769023879 for '' in 1588230740 of hbase:meta failed, tries = 9, maxAttempts = 46, timeout = 60000 ms, time elapsed = 18240 ms java.net.ConnectException: Call to address=fc3fda29f660:38897 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: fc3fda29f660/172.17.0.2:38897 at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:220) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:395) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.BufferCallBeforeInitHandler.userEventTriggered(BufferCallBeforeInitHandler.java:106) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:398) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:376) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireUserEventTriggered(AbstractChannelHandlerContext.java:368) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.userEventTriggered(DefaultChannelPipeline.java:1375) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:396) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:376) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireUserEventTriggered(DefaultChannelPipeline.java:862) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcConnection.failInit(NettyRpcConnection.java:210) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcConnection$2.fail(NettyRpcConnection.java:414) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcConnection$2.operationComplete(NettyRpcConnection.java:421) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcConnection$2.operationComplete(NettyRpcConnection.java:389) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:629) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:118) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.fulfillConnectPromise(AbstractEpollChannel.java:679) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:698) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: fc3fda29f660/172.17.0.2:38897 Caused by: java.net.ConnectException: finishConnect(..) failed: Connection refused at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.newConnectException0(Errors.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.handleConnectErrno(Errors.java:131) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Socket.finishConnect(Socket.java:359) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.doFinishConnect(AbstractEpollChannel.java:715) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:692) ~[hbase-shaded-netty-4.1.9.jar:?] ... 7 more 2024-11-16T14:59:29,843 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=fc3fda29f660,38897,1731769023879, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=fc3fda29f660,38897,1731769023879, seqNum=-1, error=java.net.ConnectException: Call to address=fc3fda29f660:38897 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: fc3fda29f660/172.17.0.2:38897 2024-11-16T14:59:29,844 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:meta,,1.1588230740, hostname=fc3fda29f660,38897,1731769023879, seqNum=-1 is java.net.ConnectException: finishConnect(..) failed: Connection refused 2024-11-16T14:59:29,844 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:meta,,1.1588230740, hostname=fc3fda29f660,38897,1731769023879, seqNum=-1 from cache 2024-11-16T14:59:29,844 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.FailedServers(52): Added failed server with address fc3fda29f660:38897 to list caused by org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: fc3fda29f660/172.17.0.2:38897 2024-11-16T14:59:32,166 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-16T14:59:39,949 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-16T14:59:39,949 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fc3fda29f660,38897,1731769023879, seqNum=-1] 2024-11-16T14:59:39,952 WARN [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.NettyRpcConnection$2(409): Exception encountered while connecting to the server fc3fda29f660:38897 org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: fc3fda29f660/172.17.0.2:38897 Caused by: java.net.ConnectException: finishConnect(..) failed: Connection refused at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.newConnectException0(Errors.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.handleConnectErrno(Errors.java:131) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Socket.finishConnect(Socket.java:359) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.doFinishConnect(AbstractEpollChannel.java:715) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:692) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:59:39,953 WARN [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRpcRetryingCaller(177): Call to fc3fda29f660,38897,1731769023879 for '' in 1588230740 of hbase:meta failed, tries = 10, maxAttempts = 46, timeout = 60000 ms, time elapsed = 28350 ms java.net.ConnectException: Call to address=fc3fda29f660:38897 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: fc3fda29f660/172.17.0.2:38897 at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:220) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:395) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.BufferCallBeforeInitHandler.userEventTriggered(BufferCallBeforeInitHandler.java:106) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:398) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:376) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireUserEventTriggered(AbstractChannelHandlerContext.java:368) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.userEventTriggered(DefaultChannelPipeline.java:1375) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:396) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:376) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireUserEventTriggered(DefaultChannelPipeline.java:862) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcConnection.failInit(NettyRpcConnection.java:210) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcConnection$2.fail(NettyRpcConnection.java:414) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcConnection$2.operationComplete(NettyRpcConnection.java:421) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcConnection$2.operationComplete(NettyRpcConnection.java:389) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:629) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:118) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.fulfillConnectPromise(AbstractEpollChannel.java:679) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:698) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: fc3fda29f660/172.17.0.2:38897 Caused by: java.net.ConnectException: finishConnect(..) failed: Connection refused at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.newConnectException0(Errors.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.handleConnectErrno(Errors.java:131) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Socket.finishConnect(Socket.java:359) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.doFinishConnect(AbstractEpollChannel.java:715) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:692) ~[hbase-shaded-netty-4.1.9.jar:?] ... 7 more 2024-11-16T14:59:39,954 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=fc3fda29f660,38897,1731769023879, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=fc3fda29f660,38897,1731769023879, seqNum=-1, error=java.net.ConnectException: Call to address=fc3fda29f660:38897 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: fc3fda29f660/172.17.0.2:38897 2024-11-16T14:59:39,954 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:meta,,1.1588230740, hostname=fc3fda29f660,38897,1731769023879, seqNum=-1 is java.net.ConnectException: finishConnect(..) failed: Connection refused 2024-11-16T14:59:39,954 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:meta,,1.1588230740, hostname=fc3fda29f660,38897,1731769023879, seqNum=-1 from cache 2024-11-16T14:59:39,955 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.FailedServers(52): Added failed server with address fc3fda29f660:38897 to list caused by org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: fc3fda29f660/172.17.0.2:38897 2024-11-16T14:59:49,999 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-16T14:59:49,999 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fc3fda29f660,38897,1731769023879, seqNum=-1] 2024-11-16T14:59:50,001 WARN [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.NettyRpcConnection$2(409): Exception encountered while connecting to the server fc3fda29f660:38897 org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: fc3fda29f660/172.17.0.2:38897 Caused by: java.net.ConnectException: finishConnect(..) failed: Connection refused at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.newConnectException0(Errors.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.handleConnectErrno(Errors.java:131) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Socket.finishConnect(Socket.java:359) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.doFinishConnect(AbstractEpollChannel.java:715) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:692) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T14:59:50,003 WARN [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRpcRetryingCaller(177): Call to fc3fda29f660,38897,1731769023879 for '' in 1588230740 of hbase:meta failed, tries = 11, maxAttempts = 46, timeout = 60000 ms, time elapsed = 38400 ms java.net.ConnectException: Call to address=fc3fda29f660:38897 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: fc3fda29f660/172.17.0.2:38897 at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:220) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:395) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.BufferCallBeforeInitHandler.userEventTriggered(BufferCallBeforeInitHandler.java:106) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:398) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:376) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireUserEventTriggered(AbstractChannelHandlerContext.java:368) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.userEventTriggered(DefaultChannelPipeline.java:1375) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:396) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:376) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireUserEventTriggered(DefaultChannelPipeline.java:862) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcConnection.failInit(NettyRpcConnection.java:210) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcConnection$2.fail(NettyRpcConnection.java:414) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcConnection$2.operationComplete(NettyRpcConnection.java:421) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcConnection$2.operationComplete(NettyRpcConnection.java:389) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:629) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:118) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.fulfillConnectPromise(AbstractEpollChannel.java:679) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:698) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: fc3fda29f660/172.17.0.2:38897 Caused by: java.net.ConnectException: finishConnect(..) failed: Connection refused at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.newConnectException0(Errors.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.handleConnectErrno(Errors.java:131) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Socket.finishConnect(Socket.java:359) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.doFinishConnect(AbstractEpollChannel.java:715) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:692) ~[hbase-shaded-netty-4.1.9.jar:?] ... 7 more 2024-11-16T14:59:50,004 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=fc3fda29f660,38897,1731769023879, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=fc3fda29f660,38897,1731769023879, seqNum=-1, error=java.net.ConnectException: Call to address=fc3fda29f660:38897 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: fc3fda29f660/172.17.0.2:38897 2024-11-16T14:59:50,004 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:meta,,1.1588230740, hostname=fc3fda29f660,38897,1731769023879, seqNum=-1 is java.net.ConnectException: finishConnect(..) failed: Connection refused 2024-11-16T14:59:50,004 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:meta,,1.1588230740, hostname=fc3fda29f660,38897,1731769023879, seqNum=-1 from cache 2024-11-16T14:59:50,005 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.FailedServers(52): Added failed server with address fc3fda29f660:38897 to list caused by org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: fc3fda29f660/172.17.0.2:38897 2024-11-16T15:00:00,029 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-16T15:00:00,029 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fc3fda29f660,38897,1731769023879, seqNum=-1] 2024-11-16T15:00:00,031 WARN [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.NettyRpcConnection$2(409): Exception encountered while connecting to the server fc3fda29f660:38897 org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: fc3fda29f660/172.17.0.2:38897 Caused by: java.net.ConnectException: finishConnect(..) failed: Connection refused at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.newConnectException0(Errors.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.handleConnectErrno(Errors.java:131) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Socket.finishConnect(Socket.java:359) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.doFinishConnect(AbstractEpollChannel.java:715) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:692) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T15:00:00,032 WARN [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRpcRetryingCaller(177): Call to fc3fda29f660,38897,1731769023879 for '' in 1588230740 of hbase:meta failed, tries = 12, maxAttempts = 46, timeout = 60000 ms, time elapsed = 48430 ms java.net.ConnectException: Call to address=fc3fda29f660:38897 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: fc3fda29f660/172.17.0.2:38897 at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:220) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:395) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.BufferCallBeforeInitHandler.userEventTriggered(BufferCallBeforeInitHandler.java:106) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:398) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:376) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireUserEventTriggered(AbstractChannelHandlerContext.java:368) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.userEventTriggered(DefaultChannelPipeline.java:1375) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:396) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:376) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireUserEventTriggered(DefaultChannelPipeline.java:862) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcConnection.failInit(NettyRpcConnection.java:210) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcConnection$2.fail(NettyRpcConnection.java:414) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcConnection$2.operationComplete(NettyRpcConnection.java:421) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcConnection$2.operationComplete(NettyRpcConnection.java:389) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:629) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:118) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.fulfillConnectPromise(AbstractEpollChannel.java:679) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:698) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: fc3fda29f660/172.17.0.2:38897 Caused by: java.net.ConnectException: finishConnect(..) failed: Connection refused at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.newConnectException0(Errors.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.handleConnectErrno(Errors.java:131) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Socket.finishConnect(Socket.java:359) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.doFinishConnect(AbstractEpollChannel.java:715) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:692) ~[hbase-shaded-netty-4.1.9.jar:?] ... 7 more 2024-11-16T15:00:00,033 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=fc3fda29f660,38897,1731769023879, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=fc3fda29f660,38897,1731769023879, seqNum=-1, error=java.net.ConnectException: Call to address=fc3fda29f660:38897 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: fc3fda29f660/172.17.0.2:38897 2024-11-16T15:00:00,034 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:meta,,1.1588230740, hostname=fc3fda29f660,38897,1731769023879, seqNum=-1 is java.net.ConnectException: finishConnect(..) failed: Connection refused 2024-11-16T15:00:00,034 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:meta,,1.1588230740, hostname=fc3fda29f660,38897,1731769023879, seqNum=-1 from cache 2024-11-16T15:00:00,034 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.FailedServers(52): Added failed server with address fc3fda29f660:38897 to list caused by org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: fc3fda29f660/172.17.0.2:38897 2024-11-16T15:00:02,102 WARN [RS:0;fc3fda29f660:39089 {}] monitoring.TaskMonitor(166): Status Processing ServerCrashProcedure of fc3fda29f660,38897,1731769023879: status=Processing ServerCrashProcedure of fc3fda29f660,38897,1731769023879 current State SERVER_CRASH_SPLIT_META_LOGS, state=RUNNING, startTime=1731769031395, completionTime=-1 appears to have been leaked 2024-11-16T15:00:02,102 WARN [RS:0;fc3fda29f660:39089 {}] monitoring.TaskMonitor(166): Status Processing ServerCrashProcedure of fc3fda29f660,38897,1731769023879: status=Processing ServerCrashProcedure of fc3fda29f660,38897,1731769023879 current State SERVER_CRASH_DELETE_SPLIT_META_WALS_DIR, state=RUNNING, startTime=1731769091429, completionTime=-1 appears to have been leaked 2024-11-16T15:00:02,166 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-16T15:00:11,610 WARN [Time-limited test {}] procedure.MasterProcedureTestingUtility$3(130): Failed to load meta org.apache.hadoop.hbase.client.RetriesExhaustedException: Failed after attempts=13, exceptions: 2024-11-16T14:59:11.604Z, java.net.ConnectException: Call to address=fc3fda29f660:38897 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: fc3fda29f660/172.17.0.2:38897 2024-11-16T14:59:11.710Z, org.apache.hadoop.hbase.ipc.FailedServerException: Call to address=fc3fda29f660:38897 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: fc3fda29f660:38897 2024-11-16T14:59:11.920Z, org.apache.hadoop.hbase.ipc.FailedServerException: Call to address=fc3fda29f660:38897 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: fc3fda29f660:38897 2024-11-16T14:59:12.231Z, org.apache.hadoop.hbase.ipc.FailedServerException: Call to address=fc3fda29f660:38897 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: fc3fda29f660:38897 2024-11-16T14:59:12.741Z, org.apache.hadoop.hbase.ipc.FailedServerException: Call to address=fc3fda29f660:38897 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: fc3fda29f660:38897 2024-11-16T14:59:13.763Z, java.net.ConnectException: Call to address=fc3fda29f660:38897 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: fc3fda29f660/172.17.0.2:38897 2024-11-16T14:59:15.774Z, java.net.ConnectException: Call to address=fc3fda29f660:38897 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: fc3fda29f660/172.17.0.2:38897 2024-11-16T14:59:19.812Z, java.net.ConnectException: Call to address=fc3fda29f660:38897 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: fc3fda29f660/172.17.0.2:38897 2024-11-16T14:59:29.844Z, java.net.ConnectException: Call to address=fc3fda29f660:38897 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: fc3fda29f660/172.17.0.2:38897 2024-11-16T14:59:39.954Z, java.net.ConnectException: Call to address=fc3fda29f660:38897 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: fc3fda29f660/172.17.0.2:38897 2024-11-16T14:59:50.004Z, java.net.ConnectException: Call to address=fc3fda29f660:38897 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: fc3fda29f660/172.17.0.2:38897 2024-11-16T15:00:00.034Z, java.net.ConnectException: Call to address=fc3fda29f660:38897 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: fc3fda29f660/172.17.0.2:38897 at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:163) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncTableResultScanner.next(AsyncTableResultScanner.java:173) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.MetaTableAccessor.scanMeta(MetaTableAccessor.java:493) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.MetaTableAccessor.scanMeta(MetaTableAccessor.java:461) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.MetaTableAccessor.scanMeta(MetaTableAccessor.java:422) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.MetaTableAccessor.fullScanRegions(MetaTableAccessor.java:108) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionStateStore.visitMeta(RegionStateStore.java:108) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.AssignmentManager.loadMeta(AssignmentManager.java:1907) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.AssignmentManager.joinCluster(AssignmentManager.java:1769) ~[classes/:?] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility$3.call(MasterProcedureTestingUtility.java:126) ~[test-classes/:?] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility$3.call(MasterProcedureTestingUtility.java:121) ~[test-classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:143) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT-tests.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT-tests.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.testRecoveryAndDoubleExecution(MasterProcedureTestingUtility.java:477) ~[test-classes/:?] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.testRecoveryAndDoubleExecution(MasterProcedureTestingUtility.java:486) ~[test-classes/:?] at org.apache.hadoop.hbase.master.procedure.TestSCPBase.testRecoveryAndDoubleExecution(TestSCPBase.java:114) ~[test-classes/:?] at org.apache.hadoop.hbase.master.procedure.TestSCPWithMeta.testRecoveryAndDoubleExecutionOnRsWithMeta(TestSCPWithMeta.java:36) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.completeExceptionally(AsyncRpcRetryingCaller.java:109) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.doCall(AsyncSingleRequestRpcRetryingCaller.java:102) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.lambda$tryScheduleRetry$1(AsyncRpcRetryingCaller.java:139) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelTimeout.run(HashedWheelTimer.java:713) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.ImmediateExecutor.execute(ImmediateExecutor.java:34) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelTimeout.expire(HashedWheelTimer.java:701) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelBucket.expireTimeouts(HashedWheelTimer.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:501) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.net.ConnectException: Call to address=fc3fda29f660:38897 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: fc3fda29f660/172.17.0.2:38897 at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:220) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:395) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.BufferCallBeforeInitHandler.userEventTriggered(BufferCallBeforeInitHandler.java:106) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:398) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:376) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireUserEventTriggered(AbstractChannelHandlerContext.java:368) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.userEventTriggered(DefaultChannelPipeline.java:1375) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:396) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:376) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireUserEventTriggered(DefaultChannelPipeline.java:862) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcConnection.failInit(NettyRpcConnection.java:210) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcConnection$2.fail(NettyRpcConnection.java:414) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcConnection$2.operationComplete(NettyRpcConnection.java:421) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcConnection$2.operationComplete(NettyRpcConnection.java:389) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:629) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:118) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.fulfillConnectPromise(AbstractEpollChannel.java:679) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:698) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] ... 1 more Caused by: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: fc3fda29f660/172.17.0.2:38897 Caused by: java.net.ConnectException: finishConnect(..) failed: Connection refused at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.newConnectException0(Errors.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.handleConnectErrno(Errors.java:131) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Socket.finishConnect(Socket.java:359) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.doFinishConnect(AbstractEpollChannel.java:715) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:692) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] ... 1 more 2024-11-16T15:00:11,613 INFO [Time-limited test {}] procedure.MasterProcedureTestingUtility(473): Restart 3 exec state=pid=57, state=WAITING:SERVER_CRASH_GET_REGIONS, hasLock=true; ServerCrashProcedure fc3fda29f660,38897,1731769023879, splitWal=true, meta=true 2024-11-16T15:00:11,613 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-16T15:00:11,613 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-16T15:00:11,613 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-16T15:00:11,613 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-16T15:00:11,614 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-16T15:00:11,614 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-16T15:00:11,614 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-16T15:00:11,615 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-16T15:00:11,616 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 1 msec 2024-11-16T15:00:11,621 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=57, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-16T15:00:11,621 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=58, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-16T15:00:11,621 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=59, ppid=58, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-16T15:00:11,622 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-16T15:00:11,623 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true 2024-11-16T15:00:11,624 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=57, state=RUNNABLE:SERVER_CRASH_ASSIGN_META, hasLock=false; ServerCrashProcedure fc3fda29f660,38897,1731769023879, splitWal=true, meta=true 2024-11-16T15:00:11,624 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure fc3fda29f660%2C38897%2C1731769023879.meta.1731769025887.meta, worker=fc3fda29f660,42985,1731769023951 2024-11-16T15:00:11,624 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure fc3fda29f660%2C38897%2C1731769023879.meta.1731769025887.meta, worker=fc3fda29f660,42985,1731769023951 2024-11-16T15:00:11,624 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=57, state=RUNNABLE:SERVER_CRASH_ASSIGN_META, hasLock=false; ServerCrashProcedure fc3fda29f660,38897,1731769023879, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-16T15:00:11,624 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure fc3fda29f660%2C38897%2C1731769023879.meta.1731769025887.meta, worker=fc3fda29f660,42985,1731769023951 didn't hold the lock before restarting, skip acquiring lock. 2024-11-16T15:00:11,624 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure fc3fda29f660%2C38897%2C1731769023879.meta.1731769025887.meta, worker=fc3fda29f660,42985,1731769023951 didn't hold the lock before restarting, skip acquiring lock. 2024-11-16T15:00:11,624 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 7 msec 2024-11-16T15:00:11,625 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=fc3fda29f660,38897,1731769023879, regionLocation=fc3fda29f660,38897,1731769023879, openSeqNum=2 2024-11-16T15:00:11,625 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=fc3fda29f660,38897,1731769023879, table=hbase:meta, region=1588230740 2024-11-16T15:00:11,625 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as fc3fda29f660,38897,1731769023879, state=OPEN 2024-11-16T15:00:11,678 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=43615 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:630) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T15:00:11,679 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=43615 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:630) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T15:00:11,712 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39089-0x101456fc9c70001, quorum=127.0.0.1:63224, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-16T15:00:11,712 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43615-0x101456fc9c70000, quorum=127.0.0.1:63224, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-16T15:00:11,712 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42985-0x101456fc9c70003, quorum=127.0.0.1:63224, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-16T15:00:11,712 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-16T15:00:11,712 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-16T15:00:11,712 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-16T15:00:11,715 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731769241715 2024-11-16T15:00:11,715 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-16T15:00:11,716 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=60, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-11-16T15:00:11,716 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(132): Toggle KILL before store update to: true 2024-11-16T15:00:11,716 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-16T15:00:11,716 DEBUG [Time-limited test {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fc3fda29f660,38897,1731769023879, seqNum=-1] 2024-11-16T15:00:11,718 WARN [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.NettyRpcConnection$2(409): Exception encountered while connecting to the server fc3fda29f660:38897 org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: fc3fda29f660/172.17.0.2:38897 Caused by: java.net.ConnectException: finishConnect(..) failed: Connection refused at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.newConnectException0(Errors.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.handleConnectErrno(Errors.java:131) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Socket.finishConnect(Socket.java:359) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.doFinishConnect(AbstractEpollChannel.java:715) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:692) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T15:00:11,719 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=fc3fda29f660,38897,1731769023879, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=fc3fda29f660,38897,1731769023879, seqNum=-1, error=java.net.ConnectException: Call to address=fc3fda29f660:38897 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: fc3fda29f660/172.17.0.2:38897 2024-11-16T15:00:11,719 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:meta,,1.1588230740, hostname=fc3fda29f660,38897,1731769023879, seqNum=-1 is java.net.ConnectException: finishConnect(..) failed: Connection refused 2024-11-16T15:00:11,719 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:meta,,1.1588230740, hostname=fc3fda29f660,38897,1731769023879, seqNum=-1 from cache 2024-11-16T15:00:11,719 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.FailedServers(52): Added failed server with address fc3fda29f660:38897 to list caused by org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: fc3fda29f660/172.17.0.2:38897 2024-11-16T15:00:11,720 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=60, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-16T15:00:11,721 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=60, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-11-16T15:00:11,829 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-16T15:00:11,829 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fc3fda29f660,38897,1731769023879, seqNum=-1] 2024-11-16T15:00:11,830 DEBUG [Async-Client-Retry-Timer-pool-0 {}] ipc.AbstractRpcClient(357): Not trying to connect to fc3fda29f660:38897 this server is in the failed servers list 2024-11-16T15:00:11,830 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=fc3fda29f660,38897,1731769023879, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=fc3fda29f660,38897,1731769023879, seqNum=-1, error=org.apache.hadoop.hbase.ipc.FailedServerException: Call to address=fc3fda29f660:38897 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: fc3fda29f660:38897 2024-11-16T15:00:11,830 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:meta,,1.1588230740, hostname=fc3fda29f660,38897,1731769023879, seqNum=-1 is org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: fc3fda29f660:38897 2024-11-16T15:00:11,830 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:meta,,1.1588230740, hostname=fc3fda29f660,38897,1731769023879, seqNum=-1 from cache 2024-11-16T15:00:11,871 DEBUG [fc3fda29f660:43615 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=2, allServersCount=2 2024-11-16T15:00:11,872 DEBUG [fc3fda29f660:43615 {}] balancer.BalancerClusterState(204): Hosts are {fc3fda29f660=0} racks are {/default-rack=0} 2024-11-16T15:00:11,873 DEBUG [fc3fda29f660:43615 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-11-16T15:00:11,873 DEBUG [fc3fda29f660:43615 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-11-16T15:00:11,873 DEBUG [fc3fda29f660:43615 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-11-16T15:00:11,873 DEBUG [fc3fda29f660:43615 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-11-16T15:00:11,873 INFO [fc3fda29f660:43615 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-11-16T15:00:11,873 INFO [fc3fda29f660:43615 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-11-16T15:00:11,873 DEBUG [fc3fda29f660:43615 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-11-16T15:00:11,874 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=60 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=fc3fda29f660,39089,1731769023769 2024-11-16T15:00:11,876 INFO [PEWorker-1 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as fc3fda29f660,39089,1731769023769, state=OPENING 2024-11-16T15:00:11,903 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43615-0x101456fc9c70000, quorum=127.0.0.1:63224, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-16T15:00:11,903 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42985-0x101456fc9c70003, quorum=127.0.0.1:63224, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-16T15:00:11,903 DEBUG [PEWorker-1 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=60, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-16T15:00:11,904 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-16T15:00:11,904 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-16T15:00:11,904 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=61, ppid=60, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=fc3fda29f660,39089,1731769023769}] 2024-11-16T15:00:11,903 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39089-0x101456fc9c70001, quorum=127.0.0.1:63224, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-16T15:00:11,904 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-16T15:00:12,039 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-16T15:00:12,039 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fc3fda29f660,39089,1731769023769, seqNum=-1] 2024-11-16T15:00:12,040 DEBUG [Async-Client-Retry-Timer-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-16T15:00:12,044 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33929, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-16T15:00:12,055 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39089 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Scan size: 121 connection: 172.17.0.2:33929 deadline: 1731769271721, exception=org.apache.hadoop.hbase.NotServingRegionException: hbase:meta,,1 is not online on fc3fda29f660,39089,1731769023769 2024-11-16T15:00:12,055 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=fc3fda29f660,39089,1731769023769, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=fc3fda29f660,39089,1731769023769, seqNum=-1, error=org.apache.hadoop.hbase.NotServingRegionException: org.apache.hadoop.hbase.NotServingRegionException: hbase:meta,,1 is not online on fc3fda29f660,39089,1731769023769 at org.apache.hadoop.hbase.regionserver.HRegionServer.getRegionByEncodedName(HRegionServer.java:3186) at org.apache.hadoop.hbase.regionserver.HRegionServer.getRegion(HRegionServer.java:3164) at org.apache.hadoop.hbase.regionserver.RSRpcServices.getRegion(RSRpcServices.java:1413) at org.apache.hadoop.hbase.regionserver.RSRpcServices.checkQuotaAndGetRegionScannerContext(RSRpcServices.java:4010) at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3577) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T15:00:12,055 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:meta,,1.1588230740, hostname=fc3fda29f660,39089,1731769023769, seqNum=-1 is org.apache.hadoop.hbase.NotServingRegionException: org.apache.hadoop.hbase.NotServingRegionException: hbase:meta,,1 is not online on fc3fda29f660,39089,1731769023769 at org.apache.hadoop.hbase.regionserver.HRegionServer.getRegionByEncodedName(HRegionServer.java:3186) at org.apache.hadoop.hbase.regionserver.HRegionServer.getRegion(HRegionServer.java:3164) at org.apache.hadoop.hbase.regionserver.RSRpcServices.getRegion(RSRpcServices.java:1413) at org.apache.hadoop.hbase.regionserver.RSRpcServices.checkQuotaAndGetRegionScannerContext(RSRpcServices.java:4010) at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3577) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T15:00:12,055 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:meta,,1.1588230740, hostname=fc3fda29f660,39089,1731769023769, seqNum=-1 from cache 2024-11-16T15:00:12,060 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45957, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-16T15:00:12,064 INFO [RS_OPEN_META-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_META, pid=61}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-11-16T15:00:12,064 INFO [RS_OPEN_META-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_META, pid=61}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-16T15:00:12,065 INFO [RS_OPEN_META-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_META, pid=61}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-11-16T15:00:12,067 INFO [RS_OPEN_META-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_META, pid=61}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=fc3fda29f660%2C39089%2C1731769023769.meta, suffix=.meta, logDir=hdfs://localhost:42223/user/jenkins/test-data/a8176cb1-2ebb-b530-b3c7-53c9e53c186e/WALs/fc3fda29f660,39089,1731769023769, archiveDir=hdfs://localhost:42223/user/jenkins/test-data/a8176cb1-2ebb-b530-b3c7-53c9e53c186e/oldWALs, maxLogs=32 2024-11-16T15:00:12,081 DEBUG [RS_OPEN_META-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_META, pid=61}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/a8176cb1-2ebb-b530-b3c7-53c9e53c186e/WALs/fc3fda29f660,39089,1731769023769/fc3fda29f660%2C39089%2C1731769023769.meta.1731769212069.meta, exclude list is [], retry=0 2024-11-16T15:00:12,085 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41671,DS-cfbd562b-a0b9-4a9e-8390-a5133b0d2c73,DISK] 2024-11-16T15:00:12,085 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36531,DS-69dfd3e9-4ae1-4c6e-b575-41b55f386dcc,DISK] 2024-11-16T15:00:12,085 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40641,DS-cecddcef-3bfd-4cf3-a998-7e060881adbc,DISK] 2024-11-16T15:00:12,087 INFO [RS_OPEN_META-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_META, pid=61}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/a8176cb1-2ebb-b530-b3c7-53c9e53c186e/WALs/fc3fda29f660,39089,1731769023769/fc3fda29f660%2C39089%2C1731769023769.meta.1731769212069.meta 2024-11-16T15:00:12,088 DEBUG [RS_OPEN_META-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_META, pid=61}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:40829:40829),(127.0.0.1/127.0.0.1:39249:39249),(127.0.0.1/127.0.0.1:35235:35235)] 2024-11-16T15:00:12,088 DEBUG [RS_OPEN_META-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-11-16T15:00:12,089 DEBUG [RS_OPEN_META-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_META, pid=61}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-11-16T15:00:12,089 DEBUG [RS_OPEN_META-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-11-16T15:00:12,089 INFO [RS_OPEN_META-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-11-16T15:00:12,089 DEBUG [RS_OPEN_META-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-11-16T15:00:12,089 DEBUG [RS_OPEN_META-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-16T15:00:12,090 DEBUG [RS_OPEN_META-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-11-16T15:00:12,090 DEBUG [RS_OPEN_META-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-11-16T15:00:12,093 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-16T15:00:12,094 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-16T15:00:12,094 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T15:00:12,095 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-16T15:00:12,095 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-16T15:00:12,096 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-16T15:00:12,096 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T15:00:12,097 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-16T15:00:12,097 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-16T15:00:12,098 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-16T15:00:12,098 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T15:00:12,098 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-16T15:00:12,099 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-16T15:00:12,100 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-16T15:00:12,100 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T15:00:12,100 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-16T15:00:12,100 DEBUG [RS_OPEN_META-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-16T15:00:12,101 DEBUG [RS_OPEN_META-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42223/user/jenkins/test-data/a8176cb1-2ebb-b530-b3c7-53c9e53c186e/data/hbase/meta/1588230740 2024-11-16T15:00:12,103 DEBUG [RS_OPEN_META-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:42223/user/jenkins/test-data/a8176cb1-2ebb-b530-b3c7-53c9e53c186e/data/hbase/meta/1588230740 2024-11-16T15:00:12,104 INFO [RS_OPEN_META-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:42223/user/jenkins/test-data/a8176cb1-2ebb-b530-b3c7-53c9e53c186e/data/hbase/meta/1588230740/recovered.edits/0000000000000000060 2024-11-16T15:00:12,120 DEBUG [RS_OPEN_META-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegion(5793): Applied 264, skipped 0, firstSequenceIdInLog=4, maxSequenceIdInLog=60, path=hdfs://localhost:42223/user/jenkins/test-data/a8176cb1-2ebb-b530-b3c7-53c9e53c186e/data/hbase/meta/1588230740/recovered.edits/0000000000000000060 2024-11-16T15:00:12,124 INFO [RS_OPEN_META-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=49.58 KB heapSize=69.98 KB 2024-11-16T15:00:12,190 DEBUG [RS_OPEN_META-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_META, pid=61}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42223/user/jenkins/test-data/a8176cb1-2ebb-b530-b3c7-53c9e53c186e/data/hbase/meta/1588230740/.tmp/info/e5a65e9067b44d21aefe5650d7dc12be is 268, key is testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,bbb,1731769026392.c3dde5f5f668d914608a2987d4f4c001./info:regioninfo/1731769027585/Put/seqid=0 2024-11-16T15:00:12,204 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41671 is added to blk_1073741866_1042 (size=58053) 2024-11-16T15:00:12,204 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40641 is added to blk_1073741866_1042 (size=58053) 2024-11-16T15:00:12,204 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36531 is added to blk_1073741866_1042 (size=58053) 2024-11-16T15:00:12,205 INFO [RS_OPEN_META-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.31 KB at sequenceid=60 (bloomFilter=true), to=hdfs://localhost:42223/user/jenkins/test-data/a8176cb1-2ebb-b530-b3c7-53c9e53c186e/data/hbase/meta/1588230740/.tmp/info/e5a65e9067b44d21aefe5650d7dc12be 2024-11-16T15:00:12,266 DEBUG [RS_OPEN_META-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_META, pid=61}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42223/user/jenkins/test-data/a8176cb1-2ebb-b530-b3c7-53c9e53c186e/data/hbase/meta/1588230740/.tmp/ns/4acbf834c84040b88c5914fc5bceec73 is 43, key is default/ns:d/1731769026140/Put/seqid=0 2024-11-16T15:00:12,274 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40641 is added to blk_1073741867_1043 (size=5153) 2024-11-16T15:00:12,274 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41671 is added to blk_1073741867_1043 (size=5153) 2024-11-16T15:00:12,274 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36531 is added to blk_1073741867_1043 (size=5153) 2024-11-16T15:00:12,275 INFO [RS_OPEN_META-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=74 B at sequenceid=60 (bloomFilter=true), to=hdfs://localhost:42223/user/jenkins/test-data/a8176cb1-2ebb-b530-b3c7-53c9e53c186e/data/hbase/meta/1588230740/.tmp/ns/4acbf834c84040b88c5914fc5bceec73 2024-11-16T15:00:12,299 DEBUG [RS_OPEN_META-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_META, pid=61}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42223/user/jenkins/test-data/a8176cb1-2ebb-b530-b3c7-53c9e53c186e/data/hbase/meta/1588230740/.tmp/table/c059d3f215dc449ea18029c17c8150a8 is 105, key is testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/table:state/1731769027886/Put/seqid=0 2024-11-16T15:00:12,308 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40641 is added to blk_1073741868_1044 (size=5620) 2024-11-16T15:00:12,309 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41671 is added to blk_1073741868_1044 (size=5620) 2024-11-16T15:00:12,309 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36531 is added to blk_1073741868_1044 (size=5620) 2024-11-16T15:00:12,311 INFO [RS_OPEN_META-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=202 B at sequenceid=60 (bloomFilter=true), to=hdfs://localhost:42223/user/jenkins/test-data/a8176cb1-2ebb-b530-b3c7-53c9e53c186e/data/hbase/meta/1588230740/.tmp/table/c059d3f215dc449ea18029c17c8150a8 2024-11-16T15:00:12,322 DEBUG [RS_OPEN_META-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42223/user/jenkins/test-data/a8176cb1-2ebb-b530-b3c7-53c9e53c186e/data/hbase/meta/1588230740/.tmp/info/e5a65e9067b44d21aefe5650d7dc12be as hdfs://localhost:42223/user/jenkins/test-data/a8176cb1-2ebb-b530-b3c7-53c9e53c186e/data/hbase/meta/1588230740/info/e5a65e9067b44d21aefe5650d7dc12be 2024-11-16T15:00:12,332 INFO [RS_OPEN_META-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42223/user/jenkins/test-data/a8176cb1-2ebb-b530-b3c7-53c9e53c186e/data/hbase/meta/1588230740/info/e5a65e9067b44d21aefe5650d7dc12be, entries=260, sequenceid=60, filesize=56.7 K 2024-11-16T15:00:12,335 DEBUG [RS_OPEN_META-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42223/user/jenkins/test-data/a8176cb1-2ebb-b530-b3c7-53c9e53c186e/data/hbase/meta/1588230740/.tmp/ns/4acbf834c84040b88c5914fc5bceec73 as hdfs://localhost:42223/user/jenkins/test-data/a8176cb1-2ebb-b530-b3c7-53c9e53c186e/data/hbase/meta/1588230740/ns/4acbf834c84040b88c5914fc5bceec73 2024-11-16T15:00:12,344 INFO [RS_OPEN_META-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42223/user/jenkins/test-data/a8176cb1-2ebb-b530-b3c7-53c9e53c186e/data/hbase/meta/1588230740/ns/4acbf834c84040b88c5914fc5bceec73, entries=2, sequenceid=60, filesize=5.0 K 2024-11-16T15:00:12,346 DEBUG [RS_OPEN_META-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42223/user/jenkins/test-data/a8176cb1-2ebb-b530-b3c7-53c9e53c186e/data/hbase/meta/1588230740/.tmp/table/c059d3f215dc449ea18029c17c8150a8 as hdfs://localhost:42223/user/jenkins/test-data/a8176cb1-2ebb-b530-b3c7-53c9e53c186e/data/hbase/meta/1588230740/table/c059d3f215dc449ea18029c17c8150a8 2024-11-16T15:00:12,355 INFO [RS_OPEN_META-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42223/user/jenkins/test-data/a8176cb1-2ebb-b530-b3c7-53c9e53c186e/data/hbase/meta/1588230740/table/c059d3f215dc449ea18029c17c8150a8, entries=2, sequenceid=60, filesize=5.5 K 2024-11-16T15:00:12,359 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-16T15:00:12,359 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fc3fda29f660,39089,1731769023769, seqNum=-1] 2024-11-16T15:00:12,360 INFO [RS_OPEN_META-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegion(3140): Finished flush of dataSize ~49.58 KB/50772, heapSize ~69.69 KB/71360, currentSize=0 B/0 for 1588230740 in 235ms, sequenceid=60, compaction requested=false; wal=null 2024-11-16T15:00:12,360 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39089 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Scan size: 121 connection: 172.17.0.2:33929 deadline: 1731769271716, exception=org.apache.hadoop.hbase.exceptions.RegionOpeningException: Region hbase:meta,,1 is opening on fc3fda29f660,39089,1731769023769 2024-11-16T15:00:12,361 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=fc3fda29f660,39089,1731769023769, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=fc3fda29f660,39089,1731769023769, seqNum=-1, error=org.apache.hadoop.hbase.exceptions.RegionOpeningException: org.apache.hadoop.hbase.exceptions.RegionOpeningException: Region hbase:meta,,1 is opening on fc3fda29f660,39089,1731769023769 at org.apache.hadoop.hbase.regionserver.HRegionServer.getRegionByEncodedName(HRegionServer.java:3183) at org.apache.hadoop.hbase.regionserver.HRegionServer.getRegion(HRegionServer.java:3164) at org.apache.hadoop.hbase.regionserver.RSRpcServices.getRegion(RSRpcServices.java:1413) at org.apache.hadoop.hbase.regionserver.RSRpcServices.checkQuotaAndGetRegionScannerContext(RSRpcServices.java:4010) at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3577) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T15:00:12,361 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:meta,,1.1588230740, hostname=fc3fda29f660,39089,1731769023769, seqNum=-1 is org.apache.hadoop.hbase.exceptions.RegionOpeningException: org.apache.hadoop.hbase.exceptions.RegionOpeningException: Region hbase:meta,,1 is opening on fc3fda29f660,39089,1731769023769 at org.apache.hadoop.hbase.regionserver.HRegionServer.getRegionByEncodedName(HRegionServer.java:3183) at org.apache.hadoop.hbase.regionserver.HRegionServer.getRegion(HRegionServer.java:3164) at org.apache.hadoop.hbase.regionserver.RSRpcServices.getRegion(RSRpcServices.java:1413) at org.apache.hadoop.hbase.regionserver.RSRpcServices.checkQuotaAndGetRegionScannerContext(RSRpcServices.java:4010) at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3577) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T15:00:12,361 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:meta,,1.1588230740, hostname=fc3fda29f660,39089,1731769023769, seqNum=-1 from cache 2024-11-16T15:00:12,362 DEBUG [RS_OPEN_META-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:42223/user/jenkins/test-data/a8176cb1-2ebb-b530-b3c7-53c9e53c186e/data/hbase/meta/1588230740/recovered.edits/0000000000000000060 2024-11-16T15:00:12,364 DEBUG [RS_OPEN_META-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-16T15:00:12,364 DEBUG [RS_OPEN_META-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-16T15:00:12,365 DEBUG [RS_OPEN_META-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-16T15:00:12,366 DEBUG [RS_OPEN_META-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-16T15:00:12,369 DEBUG [RS_OPEN_META-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_META, pid=61}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:42223/user/jenkins/test-data/a8176cb1-2ebb-b530-b3c7-53c9e53c186e/data/hbase/meta/1588230740/recovered.edits/60.seqid, newMaxSeqId=60, maxSeqId=1 2024-11-16T15:00:12,370 INFO [RS_OPEN_META-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=61; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=59966138, jitterRate=-0.1064349114894867}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-16T15:00:12,370 DEBUG [RS_OPEN_META-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-11-16T15:00:12,370 DEBUG [RS_OPEN_META-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1731769212090Writing region info on filesystem at 1731769212090Initializing all the Stores at 1731769212092 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731769212092Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731769212092Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731769212092Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731769212092Obtaining lock to block concurrent updates at 1731769212124 (+32 ms)Preparing flush snapshotting stores in 1588230740 at 1731769212124Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=50772, getHeapSize=71600, getOffHeapSize=0, getCellsCount=264 at 1731769212128 (+4 ms)Flushing stores of hbase:meta,,1.1588230740 at 1731769212128Flushing 1588230740/info: creating writer at 1731769212132 (+4 ms)Flushing 1588230740/info: appending metadata at 1731769212181 (+49 ms)Flushing 1588230740/info: closing flushed file at 1731769212184 (+3 ms)Flushing 1588230740/ns: creating writer at 1731769212250 (+66 ms)Flushing 1588230740/ns: appending metadata at 1731769212265 (+15 ms)Flushing 1588230740/ns: closing flushed file at 1731769212265Flushing 1588230740/table: creating writer at 1731769212282 (+17 ms)Flushing 1588230740/table: appending metadata at 1731769212298 (+16 ms)Flushing 1588230740/table: closing flushed file at 1731769212298Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@6c6b91b9: reopening flushed file at 1731769212321 (+23 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@7682587b: reopening flushed file at 1731769212334 (+13 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@6dc12aa8: reopening flushed file at 1731769212345 (+11 ms)Finished flush of dataSize ~49.58 KB/50772, heapSize ~69.69 KB/71360, currentSize=0 B/0 for 1588230740 in 235ms, sequenceid=60, compaction requested=false; wal=null at 1731769212360 (+15 ms)Cleaning up temporary data from old regions at 1731769212364 (+4 ms)Running coprocessor post-open hooks at 1731769212370 (+6 ms)Region opened successfully at 1731769212370 2024-11-16T15:00:12,371 INFO [RS_OPEN_META-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=61, masterSystemTime=1731769212059 2024-11-16T15:00:12,373 DEBUG [RS_OPEN_META-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-11-16T15:00:12,373 INFO [RS_OPEN_META-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_META, pid=61}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-11-16T15:00:12,374 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=60 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=61, regionLocation=fc3fda29f660,39089,1731769023769 2024-11-16T15:00:12,375 INFO [PEWorker-1 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as fc3fda29f660,39089,1731769023769, state=OPEN 2024-11-16T15:00:12,437 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43615-0x101456fc9c70000, quorum=127.0.0.1:63224, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-16T15:00:12,437 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42985-0x101456fc9c70003, quorum=127.0.0.1:63224, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-16T15:00:12,437 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39089-0x101456fc9c70001, quorum=127.0.0.1:63224, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-16T15:00:12,437 DEBUG [PEWorker-1 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=61, ppid=60, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=fc3fda29f660,39089,1731769023769 2024-11-16T15:00:12,437 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-16T15:00:12,437 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-16T15:00:12,437 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-16T15:00:12,440 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=61, resume processing ppid=60 2024-11-16T15:00:12,440 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=fc3fda29f660,39089,1731769023769 in 533 msec 2024-11-16T15:00:12,442 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=60, resume processing ppid=57 2024-11-16T15:00:12,442 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 724 msec 2024-11-16T15:00:12,869 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-16T15:00:12,869 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fc3fda29f660,39089,1731769023769, seqNum=-1] 2024-11-16T15:00:12,902 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4f0d635e77a1e3dd04c4d107bb0f1564, regionState=OPEN, lastHost=fc3fda29f660,38897,1731769023879, regionLocation=fc3fda29f660,38897,1731769023879, openSeqNum=2 2024-11-16T15:00:12,902 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c3dde5f5f668d914608a2987d4f4c001, regionState=OPEN, lastHost=fc3fda29f660,42985,1731769023951, regionLocation=fc3fda29f660,42985,1731769023951, openSeqNum=2 2024-11-16T15:00:12,902 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5bb22144956dcd43c9b1d957232b11e7, regionState=OPEN, lastHost=fc3fda29f660,39089,1731769023769, regionLocation=fc3fda29f660,39089,1731769023769, openSeqNum=2 2024-11-16T15:00:12,902 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=dde7f54607f11c5fb4271e361e82dabd, regionState=OPEN, lastHost=fc3fda29f660,39089,1731769023769, regionLocation=fc3fda29f660,39089,1731769023769, openSeqNum=2 2024-11-16T15:00:12,902 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=fd3944fc2b78246a853fb2232c747c35, regionState=OPEN, lastHost=fc3fda29f660,38897,1731769023879, regionLocation=fc3fda29f660,38897,1731769023879, openSeqNum=2 2024-11-16T15:00:12,902 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a5db9431e22152bd2a82e84504d2879a, regionState=OPEN, lastHost=fc3fda29f660,42985,1731769023951, regionLocation=fc3fda29f660,42985,1731769023951, openSeqNum=2 2024-11-16T15:00:12,902 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=dcd6414ddebad66e87ab37a6dba924f3, regionState=OPEN, lastHost=fc3fda29f660,42985,1731769023951, regionLocation=fc3fda29f660,42985,1731769023951, openSeqNum=2 2024-11-16T15:00:12,902 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1d97df67e5110b0fe1a1e4b7c1341cbb, regionState=OPEN, lastHost=fc3fda29f660,42985,1731769023951, regionLocation=fc3fda29f660,42985,1731769023951, openSeqNum=2 2024-11-16T15:00:12,903 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7352dcdbf9811dcebc51a714a1de4f55, regionState=OPEN, lastHost=fc3fda29f660,38897,1731769023879, regionLocation=fc3fda29f660,38897,1731769023879, openSeqNum=2 2024-11-16T15:00:12,903 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7eb7b75d28b808c5c3f2f96e407b4347, regionState=OPEN, lastHost=fc3fda29f660,38897,1731769023879, regionLocation=fc3fda29f660,38897,1731769023879, openSeqNum=2 2024-11-16T15:00:12,903 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b4f73326a2a6a279d503e557c6ed452f, regionState=OPEN, lastHost=fc3fda29f660,39089,1731769023769, regionLocation=fc3fda29f660,39089,1731769023769, openSeqNum=2 2024-11-16T15:00:12,903 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=266565e60d7419a2a30ceec1b0f6519b, regionState=OPEN, lastHost=fc3fda29f660,42985,1731769023951, regionLocation=fc3fda29f660,42985,1731769023951, openSeqNum=2 2024-11-16T15:00:12,903 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c2acbc05ae7bb1607a5719b701b40383, regionState=OPEN, lastHost=fc3fda29f660,38897,1731769023879, regionLocation=fc3fda29f660,38897,1731769023879, openSeqNum=2 2024-11-16T15:00:12,903 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=994ca1526cf6c4838e65f85444853f1a, regionState=OPEN, lastHost=fc3fda29f660,39089,1731769023769, regionLocation=fc3fda29f660,39089,1731769023769, openSeqNum=2 2024-11-16T15:00:12,903 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=299131947f15f9c44bcfe22cd6623096, regionState=OPEN, lastHost=fc3fda29f660,42985,1731769023951, regionLocation=fc3fda29f660,42985,1731769023951, openSeqNum=2 2024-11-16T15:00:12,903 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7f5c9df3a544933227f58b052f743f72, regionState=OPEN, lastHost=fc3fda29f660,38897,1731769023879, regionLocation=fc3fda29f660,38897,1731769023879, openSeqNum=2 2024-11-16T15:00:12,903 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=18f03e2509a3a89e4a69ba87fad73940, regionState=OPEN, lastHost=fc3fda29f660,39089,1731769023769, regionLocation=fc3fda29f660,39089,1731769023769, openSeqNum=2 2024-11-16T15:00:12,904 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b364a006589fe0d2474ffa45692a6efc, regionState=OPEN, lastHost=fc3fda29f660,42985,1731769023951, regionLocation=fc3fda29f660,42985,1731769023951, openSeqNum=2 2024-11-16T15:00:12,904 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=33054df03c1be51dc860405880be6075, regionState=OPEN, lastHost=fc3fda29f660,38897,1731769023879, regionLocation=fc3fda29f660,38897,1731769023879, openSeqNum=2 2024-11-16T15:00:12,904 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7f11f79823720d664105d8643dfa862d, regionState=OPEN, lastHost=fc3fda29f660,39089,1731769023769, regionLocation=fc3fda29f660,39089,1731769023769, openSeqNum=2 2024-11-16T15:00:12,904 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=35ef9a7f35bb0b08ec1d53b82d3f9777, regionState=OPEN, lastHost=fc3fda29f660,42985,1731769023951, regionLocation=fc3fda29f660,42985,1731769023951, openSeqNum=2 2024-11-16T15:00:12,904 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b5566c7e10a0e001c7f58316bdd43c2f, regionState=OPEN, lastHost=fc3fda29f660,38897,1731769023879, regionLocation=fc3fda29f660,38897,1731769023879, openSeqNum=2 2024-11-16T15:00:12,904 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=62a6d124ad9d161782c0632102c766de, regionState=OPEN, lastHost=fc3fda29f660,39089,1731769023769, regionLocation=fc3fda29f660,39089,1731769023769, openSeqNum=2 2024-11-16T15:00:12,904 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=dc4fcaece350cd66f6bf88cfbdff6f15, regionState=OPEN, lastHost=fc3fda29f660,42985,1731769023951, regionLocation=fc3fda29f660,42985,1731769023951, openSeqNum=2 2024-11-16T15:00:12,904 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=426ff231e487ce9f8492776ca1dacc85, regionState=OPEN, lastHost=fc3fda29f660,38897,1731769023879, regionLocation=fc3fda29f660,38897,1731769023879, openSeqNum=2 2024-11-16T15:00:12,904 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2f41a8c996224c818e917389bd3f5141, regionState=OPEN, lastHost=fc3fda29f660,39089,1731769023769, regionLocation=fc3fda29f660,39089,1731769023769, openSeqNum=2 2024-11-16T15:00:12,904 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-11-16T15:00:12,904 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731769272904 2024-11-16T15:00:12,904 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731769332904 2024-11-16T15:00:12,905 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 1.1890 sec 2024-11-16T15:00:12,905 INFO [PEWorker-1 {}] procedure.ServerCrashProcedure(207): fc3fda29f660,38897,1731769023879 had 10 regions 2024-11-16T15:00:12,905 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(132): Toggle KILL before store update to: false 2024-11-16T15:00:12,905 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update: pid=57, state=RUNNABLE:SERVER_CRASH_SPLIT_LOGS, hasLock=true; ServerCrashProcedure fc3fda29f660,38897,1731769023879, splitWal=true, meta=true 2024-11-16T15:00:12,905 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-16T15:00:12,905 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update: pid=57, state=RUNNABLE:SERVER_CRASH_SPLIT_LOGS, hasLock=true; ServerCrashProcedure fc3fda29f660,38897,1731769023879, splitWal=true, meta=true at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1879) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T15:00:13,155 INFO [Time-limited test {}] procedure.MasterProcedureTestingUtility(473): Restart 4 exec state=pid=57, state=RUNNABLE:SERVER_CRASH_SPLIT_LOGS, hasLock=true; ServerCrashProcedure fc3fda29f660,38897,1731769023879, splitWal=true, meta=true 2024-11-16T15:00:13,155 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-16T15:00:13,155 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-16T15:00:13,155 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-16T15:00:13,155 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-16T15:00:13,156 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-16T15:00:13,156 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-16T15:00:13,156 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-16T15:00:13,156 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-16T15:00:13,157 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-16T15:00:13,158 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=57, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-16T15:00:13,159 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=58, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-16T15:00:13,159 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=59, ppid=58, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-16T15:00:13,159 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-16T15:00:13,159 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-16T15:00:13,159 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-16T15:00:13,159 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true 2024-11-16T15:00:13,160 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=57, state=RUNNABLE:SERVER_CRASH_GET_REGIONS, hasLock=false; ServerCrashProcedure fc3fda29f660,38897,1731769023879, splitWal=true, meta=true 2024-11-16T15:00:13,160 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure fc3fda29f660%2C38897%2C1731769023879.meta.1731769025887.meta, worker=fc3fda29f660,42985,1731769023951 2024-11-16T15:00:13,160 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure fc3fda29f660%2C38897%2C1731769023879.meta.1731769025887.meta, worker=fc3fda29f660,42985,1731769023951 2024-11-16T15:00:13,160 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-16T15:00:13,160 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=fc3fda29f660,39089,1731769023769 2024-11-16T15:00:13,161 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=57, state=RUNNABLE:SERVER_CRASH_GET_REGIONS, hasLock=false; ServerCrashProcedure fc3fda29f660,38897,1731769023879, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-16T15:00:13,161 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure fc3fda29f660%2C38897%2C1731769023879.meta.1731769025887.meta, worker=fc3fda29f660,42985,1731769023951 didn't hold the lock before restarting, skip acquiring lock. 2024-11-16T15:00:13,161 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure fc3fda29f660%2C38897%2C1731769023879.meta.1731769025887.meta, worker=fc3fda29f660,42985,1731769023951 didn't hold the lock before restarting, skip acquiring lock. 2024-11-16T15:00:13,161 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-16T15:00:13,161 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=fc3fda29f660,39089,1731769023769 didn't hold the lock before restarting, skip acquiring lock. 2024-11-16T15:00:13,161 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 4 msec 2024-11-16T15:00:13,162 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=fc3fda29f660,39089,1731769023769, regionLocation=fc3fda29f660,39089,1731769023769, openSeqNum=61 2024-11-16T15:00:13,162 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=fc3fda29f660,39089,1731769023769, table=hbase:meta, region=1588230740 2024-11-16T15:00:13,162 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as fc3fda29f660,39089,1731769023769, state=OPEN 2024-11-16T15:00:13,212 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43615-0x101456fc9c70000, quorum=127.0.0.1:63224, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-16T15:00:13,212 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39089-0x101456fc9c70001, quorum=127.0.0.1:63224, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-16T15:00:13,212 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42985-0x101456fc9c70003, quorum=127.0.0.1:63224, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-16T15:00:13,212 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-16T15:00:13,212 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-16T15:00:13,212 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-16T15:00:13,213 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731769243213 2024-11-16T15:00:13,213 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-16T15:00:13,220 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4f0d635e77a1e3dd04c4d107bb0f1564, regionState=OPEN, lastHost=fc3fda29f660,38897,1731769023879, regionLocation=fc3fda29f660,38897,1731769023879, openSeqNum=2 2024-11-16T15:00:13,220 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c3dde5f5f668d914608a2987d4f4c001, regionState=OPEN, lastHost=fc3fda29f660,42985,1731769023951, regionLocation=fc3fda29f660,42985,1731769023951, openSeqNum=2 2024-11-16T15:00:13,220 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5bb22144956dcd43c9b1d957232b11e7, regionState=OPEN, lastHost=fc3fda29f660,39089,1731769023769, regionLocation=fc3fda29f660,39089,1731769023769, openSeqNum=2 2024-11-16T15:00:13,221 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=dde7f54607f11c5fb4271e361e82dabd, regionState=OPEN, lastHost=fc3fda29f660,39089,1731769023769, regionLocation=fc3fda29f660,39089,1731769023769, openSeqNum=2 2024-11-16T15:00:13,221 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=fd3944fc2b78246a853fb2232c747c35, regionState=OPEN, lastHost=fc3fda29f660,38897,1731769023879, regionLocation=fc3fda29f660,38897,1731769023879, openSeqNum=2 2024-11-16T15:00:13,221 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a5db9431e22152bd2a82e84504d2879a, regionState=OPEN, lastHost=fc3fda29f660,42985,1731769023951, regionLocation=fc3fda29f660,42985,1731769023951, openSeqNum=2 2024-11-16T15:00:13,221 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=dcd6414ddebad66e87ab37a6dba924f3, regionState=OPEN, lastHost=fc3fda29f660,42985,1731769023951, regionLocation=fc3fda29f660,42985,1731769023951, openSeqNum=2 2024-11-16T15:00:13,221 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1d97df67e5110b0fe1a1e4b7c1341cbb, regionState=OPEN, lastHost=fc3fda29f660,42985,1731769023951, regionLocation=fc3fda29f660,42985,1731769023951, openSeqNum=2 2024-11-16T15:00:13,221 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7352dcdbf9811dcebc51a714a1de4f55, regionState=OPEN, lastHost=fc3fda29f660,38897,1731769023879, regionLocation=fc3fda29f660,38897,1731769023879, openSeqNum=2 2024-11-16T15:00:13,222 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7eb7b75d28b808c5c3f2f96e407b4347, regionState=OPEN, lastHost=fc3fda29f660,38897,1731769023879, regionLocation=fc3fda29f660,38897,1731769023879, openSeqNum=2 2024-11-16T15:00:13,222 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b4f73326a2a6a279d503e557c6ed452f, regionState=OPEN, lastHost=fc3fda29f660,39089,1731769023769, regionLocation=fc3fda29f660,39089,1731769023769, openSeqNum=2 2024-11-16T15:00:13,222 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=266565e60d7419a2a30ceec1b0f6519b, regionState=OPEN, lastHost=fc3fda29f660,42985,1731769023951, regionLocation=fc3fda29f660,42985,1731769023951, openSeqNum=2 2024-11-16T15:00:13,222 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c2acbc05ae7bb1607a5719b701b40383, regionState=OPEN, lastHost=fc3fda29f660,38897,1731769023879, regionLocation=fc3fda29f660,38897,1731769023879, openSeqNum=2 2024-11-16T15:00:13,222 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=994ca1526cf6c4838e65f85444853f1a, regionState=OPEN, lastHost=fc3fda29f660,39089,1731769023769, regionLocation=fc3fda29f660,39089,1731769023769, openSeqNum=2 2024-11-16T15:00:13,222 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=299131947f15f9c44bcfe22cd6623096, regionState=OPEN, lastHost=fc3fda29f660,42985,1731769023951, regionLocation=fc3fda29f660,42985,1731769023951, openSeqNum=2 2024-11-16T15:00:13,222 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7f5c9df3a544933227f58b052f743f72, regionState=OPEN, lastHost=fc3fda29f660,38897,1731769023879, regionLocation=fc3fda29f660,38897,1731769023879, openSeqNum=2 2024-11-16T15:00:13,222 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=18f03e2509a3a89e4a69ba87fad73940, regionState=OPEN, lastHost=fc3fda29f660,39089,1731769023769, regionLocation=fc3fda29f660,39089,1731769023769, openSeqNum=2 2024-11-16T15:00:13,223 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b364a006589fe0d2474ffa45692a6efc, regionState=OPEN, lastHost=fc3fda29f660,42985,1731769023951, regionLocation=fc3fda29f660,42985,1731769023951, openSeqNum=2 2024-11-16T15:00:13,223 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=33054df03c1be51dc860405880be6075, regionState=OPEN, lastHost=fc3fda29f660,38897,1731769023879, regionLocation=fc3fda29f660,38897,1731769023879, openSeqNum=2 2024-11-16T15:00:13,223 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7f11f79823720d664105d8643dfa862d, regionState=OPEN, lastHost=fc3fda29f660,39089,1731769023769, regionLocation=fc3fda29f660,39089,1731769023769, openSeqNum=2 2024-11-16T15:00:13,223 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=35ef9a7f35bb0b08ec1d53b82d3f9777, regionState=OPEN, lastHost=fc3fda29f660,42985,1731769023951, regionLocation=fc3fda29f660,42985,1731769023951, openSeqNum=2 2024-11-16T15:00:13,223 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b5566c7e10a0e001c7f58316bdd43c2f, regionState=OPEN, lastHost=fc3fda29f660,38897,1731769023879, regionLocation=fc3fda29f660,38897,1731769023879, openSeqNum=2 2024-11-16T15:00:13,223 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=62a6d124ad9d161782c0632102c766de, regionState=OPEN, lastHost=fc3fda29f660,39089,1731769023769, regionLocation=fc3fda29f660,39089,1731769023769, openSeqNum=2 2024-11-16T15:00:13,223 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=dc4fcaece350cd66f6bf88cfbdff6f15, regionState=OPEN, lastHost=fc3fda29f660,42985,1731769023951, regionLocation=fc3fda29f660,42985,1731769023951, openSeqNum=2 2024-11-16T15:00:13,223 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=426ff231e487ce9f8492776ca1dacc85, regionState=OPEN, lastHost=fc3fda29f660,38897,1731769023879, regionLocation=fc3fda29f660,38897,1731769023879, openSeqNum=2 2024-11-16T15:00:13,223 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2f41a8c996224c818e917389bd3f5141, regionState=OPEN, lastHost=fc3fda29f660,39089,1731769023769, regionLocation=fc3fda29f660,39089,1731769023769, openSeqNum=2 2024-11-16T15:00:13,223 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-11-16T15:00:13,223 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731769273223 2024-11-16T15:00:13,224 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731769333224 2024-11-16T15:00:13,224 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 10 msec 2024-11-16T15:00:13,224 INFO [PEWorker-1 {}] procedure.ServerCrashProcedure(207): fc3fda29f660,38897,1731769023879 had 9 regions 2024-11-16T15:00:13,224 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(132): Toggle KILL before store update to: true 2024-11-16T15:00:13,225 INFO [PEWorker-1 {}] procedure.ServerCrashProcedure(339): Splitting WALs pid=57, state=RUNNABLE:SERVER_CRASH_SPLIT_LOGS, hasLock=true; ServerCrashProcedure fc3fda29f660,38897,1731769023879, splitWal=true, meta=true, isMeta: false 2024-11-16T15:00:13,227 INFO [PEWorker-1 {}] master.SplitWALManager(105): fc3fda29f660,38897,1731769023879 WAL count=1, meta=false 2024-11-16T15:00:13,228 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=62, ppid=57, state=RUNNABLE:ACQUIRE_SPLIT_WAL_WORKER, hasLock=false; SplitWALProcedure fc3fda29f660%2C38897%2C1731769023879.1731769025434}] 2024-11-16T15:00:13,228 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(132): Toggle KILL before store update to: false 2024-11-16T15:00:13,228 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update: pid=57, state=WAITING:SERVER_CRASH_DELETE_SPLIT_WALS_DIR, hasLock=true; ServerCrashProcedure fc3fda29f660,38897,1731769023879, splitWal=true, meta=true 2024-11-16T15:00:13,228 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-16T15:00:13,228 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update: pid=57, state=WAITING:SERVER_CRASH_DELETE_SPLIT_WALS_DIR, hasLock=true; ServerCrashProcedure fc3fda29f660,38897,1731769023879, splitWal=true, meta=true at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1879) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T15:00:13,474 INFO [Time-limited test {}] procedure.MasterProcedureTestingUtility(473): Restart 5 exec state=pid=57, state=WAITING:SERVER_CRASH_DELETE_SPLIT_WALS_DIR, hasLock=true; ServerCrashProcedure fc3fda29f660,38897,1731769023879, splitWal=true, meta=true 2024-11-16T15:00:13,474 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-16T15:00:13,474 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-16T15:00:13,474 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-16T15:00:13,474 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-16T15:00:13,475 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-16T15:00:13,475 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-16T15:00:13,475 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-16T15:00:13,475 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-16T15:00:13,476 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-16T15:00:13,477 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=57, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-16T15:00:13,477 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=58, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-16T15:00:13,477 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=59, ppid=58, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-16T15:00:13,477 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-16T15:00:13,477 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-16T15:00:13,477 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-16T15:00:13,478 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true 2024-11-16T15:00:13,478 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=57, state=RUNNABLE:SERVER_CRASH_SPLIT_LOGS, hasLock=false; ServerCrashProcedure fc3fda29f660,38897,1731769023879, splitWal=true, meta=true 2024-11-16T15:00:13,478 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure fc3fda29f660%2C38897%2C1731769023879.meta.1731769025887.meta, worker=fc3fda29f660,42985,1731769023951 2024-11-16T15:00:13,478 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure fc3fda29f660%2C38897%2C1731769023879.meta.1731769025887.meta, worker=fc3fda29f660,42985,1731769023951 2024-11-16T15:00:13,479 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-16T15:00:13,479 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=fc3fda29f660,39089,1731769023769 2024-11-16T15:00:13,479 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=57, state=RUNNABLE:SERVER_CRASH_SPLIT_LOGS, hasLock=false; ServerCrashProcedure fc3fda29f660,38897,1731769023879, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-16T15:00:13,479 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure fc3fda29f660%2C38897%2C1731769023879.meta.1731769025887.meta, worker=fc3fda29f660,42985,1731769023951 didn't hold the lock before restarting, skip acquiring lock. 2024-11-16T15:00:13,479 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure fc3fda29f660%2C38897%2C1731769023879.meta.1731769025887.meta, worker=fc3fda29f660,42985,1731769023951 didn't hold the lock before restarting, skip acquiring lock. 2024-11-16T15:00:13,479 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-16T15:00:13,479 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=fc3fda29f660,39089,1731769023769 didn't hold the lock before restarting, skip acquiring lock. 2024-11-16T15:00:13,479 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 3 msec 2024-11-16T15:00:13,480 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=fc3fda29f660,39089,1731769023769, regionLocation=fc3fda29f660,39089,1731769023769, openSeqNum=61 2024-11-16T15:00:13,480 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=fc3fda29f660,39089,1731769023769, table=hbase:meta, region=1588230740 2024-11-16T15:00:13,480 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as fc3fda29f660,39089,1731769023769, state=OPEN 2024-11-16T15:00:13,512 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39089-0x101456fc9c70001, quorum=127.0.0.1:63224, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-16T15:00:13,512 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42985-0x101456fc9c70003, quorum=127.0.0.1:63224, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-16T15:00:13,512 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43615-0x101456fc9c70000, quorum=127.0.0.1:63224, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-16T15:00:13,512 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-16T15:00:13,512 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-16T15:00:13,512 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-16T15:00:13,513 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731769243513 2024-11-16T15:00:13,513 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-16T15:00:13,518 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4f0d635e77a1e3dd04c4d107bb0f1564, regionState=OPEN, lastHost=fc3fda29f660,38897,1731769023879, regionLocation=fc3fda29f660,38897,1731769023879, openSeqNum=2 2024-11-16T15:00:13,518 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c3dde5f5f668d914608a2987d4f4c001, regionState=OPEN, lastHost=fc3fda29f660,42985,1731769023951, regionLocation=fc3fda29f660,42985,1731769023951, openSeqNum=2 2024-11-16T15:00:13,518 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5bb22144956dcd43c9b1d957232b11e7, regionState=OPEN, lastHost=fc3fda29f660,39089,1731769023769, regionLocation=fc3fda29f660,39089,1731769023769, openSeqNum=2 2024-11-16T15:00:13,519 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=dde7f54607f11c5fb4271e361e82dabd, regionState=OPEN, lastHost=fc3fda29f660,39089,1731769023769, regionLocation=fc3fda29f660,39089,1731769023769, openSeqNum=2 2024-11-16T15:00:13,519 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=fd3944fc2b78246a853fb2232c747c35, regionState=OPEN, lastHost=fc3fda29f660,38897,1731769023879, regionLocation=fc3fda29f660,38897,1731769023879, openSeqNum=2 2024-11-16T15:00:13,519 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a5db9431e22152bd2a82e84504d2879a, regionState=OPEN, lastHost=fc3fda29f660,42985,1731769023951, regionLocation=fc3fda29f660,42985,1731769023951, openSeqNum=2 2024-11-16T15:00:13,519 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=dcd6414ddebad66e87ab37a6dba924f3, regionState=OPEN, lastHost=fc3fda29f660,42985,1731769023951, regionLocation=fc3fda29f660,42985,1731769023951, openSeqNum=2 2024-11-16T15:00:13,519 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1d97df67e5110b0fe1a1e4b7c1341cbb, regionState=OPEN, lastHost=fc3fda29f660,42985,1731769023951, regionLocation=fc3fda29f660,42985,1731769023951, openSeqNum=2 2024-11-16T15:00:13,519 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7352dcdbf9811dcebc51a714a1de4f55, regionState=OPEN, lastHost=fc3fda29f660,38897,1731769023879, regionLocation=fc3fda29f660,38897,1731769023879, openSeqNum=2 2024-11-16T15:00:13,519 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7eb7b75d28b808c5c3f2f96e407b4347, regionState=OPEN, lastHost=fc3fda29f660,38897,1731769023879, regionLocation=fc3fda29f660,38897,1731769023879, openSeqNum=2 2024-11-16T15:00:13,520 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b4f73326a2a6a279d503e557c6ed452f, regionState=OPEN, lastHost=fc3fda29f660,39089,1731769023769, regionLocation=fc3fda29f660,39089,1731769023769, openSeqNum=2 2024-11-16T15:00:13,520 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=266565e60d7419a2a30ceec1b0f6519b, regionState=OPEN, lastHost=fc3fda29f660,42985,1731769023951, regionLocation=fc3fda29f660,42985,1731769023951, openSeqNum=2 2024-11-16T15:00:13,520 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c2acbc05ae7bb1607a5719b701b40383, regionState=OPEN, lastHost=fc3fda29f660,38897,1731769023879, regionLocation=fc3fda29f660,38897,1731769023879, openSeqNum=2 2024-11-16T15:00:13,520 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=994ca1526cf6c4838e65f85444853f1a, regionState=OPEN, lastHost=fc3fda29f660,39089,1731769023769, regionLocation=fc3fda29f660,39089,1731769023769, openSeqNum=2 2024-11-16T15:00:13,520 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=299131947f15f9c44bcfe22cd6623096, regionState=OPEN, lastHost=fc3fda29f660,42985,1731769023951, regionLocation=fc3fda29f660,42985,1731769023951, openSeqNum=2 2024-11-16T15:00:13,520 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7f5c9df3a544933227f58b052f743f72, regionState=OPEN, lastHost=fc3fda29f660,38897,1731769023879, regionLocation=fc3fda29f660,38897,1731769023879, openSeqNum=2 2024-11-16T15:00:13,520 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=18f03e2509a3a89e4a69ba87fad73940, regionState=OPEN, lastHost=fc3fda29f660,39089,1731769023769, regionLocation=fc3fda29f660,39089,1731769023769, openSeqNum=2 2024-11-16T15:00:13,520 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b364a006589fe0d2474ffa45692a6efc, regionState=OPEN, lastHost=fc3fda29f660,42985,1731769023951, regionLocation=fc3fda29f660,42985,1731769023951, openSeqNum=2 2024-11-16T15:00:13,520 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=33054df03c1be51dc860405880be6075, regionState=OPEN, lastHost=fc3fda29f660,38897,1731769023879, regionLocation=fc3fda29f660,38897,1731769023879, openSeqNum=2 2024-11-16T15:00:13,520 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7f11f79823720d664105d8643dfa862d, regionState=OPEN, lastHost=fc3fda29f660,39089,1731769023769, regionLocation=fc3fda29f660,39089,1731769023769, openSeqNum=2 2024-11-16T15:00:13,520 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=35ef9a7f35bb0b08ec1d53b82d3f9777, regionState=OPEN, lastHost=fc3fda29f660,42985,1731769023951, regionLocation=fc3fda29f660,42985,1731769023951, openSeqNum=2 2024-11-16T15:00:13,521 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b5566c7e10a0e001c7f58316bdd43c2f, regionState=OPEN, lastHost=fc3fda29f660,38897,1731769023879, regionLocation=fc3fda29f660,38897,1731769023879, openSeqNum=2 2024-11-16T15:00:13,521 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=62a6d124ad9d161782c0632102c766de, regionState=OPEN, lastHost=fc3fda29f660,39089,1731769023769, regionLocation=fc3fda29f660,39089,1731769023769, openSeqNum=2 2024-11-16T15:00:13,521 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=dc4fcaece350cd66f6bf88cfbdff6f15, regionState=OPEN, lastHost=fc3fda29f660,42985,1731769023951, regionLocation=fc3fda29f660,42985,1731769023951, openSeqNum=2 2024-11-16T15:00:13,521 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=426ff231e487ce9f8492776ca1dacc85, regionState=OPEN, lastHost=fc3fda29f660,38897,1731769023879, regionLocation=fc3fda29f660,38897,1731769023879, openSeqNum=2 2024-11-16T15:00:13,521 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2f41a8c996224c818e917389bd3f5141, regionState=OPEN, lastHost=fc3fda29f660,39089,1731769023769, regionLocation=fc3fda29f660,39089,1731769023769, openSeqNum=2 2024-11-16T15:00:13,521 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-11-16T15:00:13,521 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731769273521 2024-11-16T15:00:13,521 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731769333521 2024-11-16T15:00:13,521 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 7 msec 2024-11-16T15:00:13,521 INFO [PEWorker-1 {}] procedure.ServerCrashProcedure(339): Splitting WALs pid=57, state=RUNNABLE:SERVER_CRASH_SPLIT_LOGS, hasLock=true; ServerCrashProcedure fc3fda29f660,38897,1731769023879, splitWal=true, meta=true, isMeta: false 2024-11-16T15:00:13,523 INFO [PEWorker-1 {}] master.SplitWALManager(105): fc3fda29f660,38897,1731769023879 WAL count=1, meta=false 2024-11-16T15:00:13,524 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=62, ppid=57, state=RUNNABLE:ACQUIRE_SPLIT_WAL_WORKER, hasLock=false; SplitWALProcedure fc3fda29f660%2C38897%2C1731769023879.1731769025434}] 2024-11-16T15:00:13,524 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(132): Toggle KILL before store update to: true 2024-11-16T15:00:13,526 DEBUG [PEWorker-1 {}] master.SplitWALManager(158): Acquired split WAL worker=fc3fda29f660,39089,1731769023769 2024-11-16T15:00:13,527 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=63, ppid=62, state=RUNNABLE, hasLock=false; SplitWALRemoteProcedure fc3fda29f660%2C38897%2C1731769023879.1731769025434, worker=fc3fda29f660,39089,1731769023769}] 2024-11-16T15:00:13,680 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39089 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SplitWALCallable, pid=63 2024-11-16T15:00:13,695 INFO [RS_LOG_REPLAY_OPS-regionserver/fc3fda29f660:0-0 {event_type=RS_LOG_REPLAY, pid=63}] wal.WALSplitter(299): Splitting hdfs://localhost:42223/user/jenkins/test-data/a8176cb1-2ebb-b530-b3c7-53c9e53c186e/WALs/fc3fda29f660,38897,1731769023879-splitting/fc3fda29f660%2C38897%2C1731769023879.1731769025434, size=327.1 K (334915bytes) 2024-11-16T15:00:13,695 INFO [RS_LOG_REPLAY_OPS-regionserver/fc3fda29f660:0-0 {event_type=RS_LOG_REPLAY, pid=63}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:42223/user/jenkins/test-data/a8176cb1-2ebb-b530-b3c7-53c9e53c186e/WALs/fc3fda29f660,38897,1731769023879-splitting/fc3fda29f660%2C38897%2C1731769023879.1731769025434 2024-11-16T15:00:13,696 INFO [RS_LOG_REPLAY_OPS-regionserver/fc3fda29f660:0-0 {event_type=RS_LOG_REPLAY, pid=63}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:42223/user/jenkins/test-data/a8176cb1-2ebb-b530-b3c7-53c9e53c186e/WALs/fc3fda29f660,38897,1731769023879-splitting/fc3fda29f660%2C38897%2C1731769023879.1731769025434 after 1ms 2024-11-16T15:00:13,700 INFO [RS_LOG_REPLAY_OPS-regionserver/fc3fda29f660:0-0 {event_type=RS_LOG_REPLAY, pid=63}] wal.WALSplitter(310): Open hdfs://localhost:42223/user/jenkins/test-data/a8176cb1-2ebb-b530-b3c7-53c9e53c186e/WALs/fc3fda29f660,38897,1731769023879-splitting/fc3fda29f660%2C38897%2C1731769023879.1731769025434 took 5ms 2024-11-16T15:00:13,703 DEBUG [RS_LOG_REPLAY_OPS-regionserver/fc3fda29f660:0-0 {event_type=RS_LOG_REPLAY, pid=63}] wal.WALSplitter(352): Last flushed sequenceid for c2acbc05ae7bb1607a5719b701b40383: last_flushed_sequence_id: 18446744073709551615 2024-11-16T15:00:13,705 DEBUG [RS_LOG_REPLAY_OPS-regionserver/fc3fda29f660:0-0 {event_type=RS_LOG_REPLAY, pid=63}] wal.WALSplitter(352): Last flushed sequenceid for 33054df03c1be51dc860405880be6075: last_flushed_sequence_id: 18446744073709551615 2024-11-16T15:00:13,706 DEBUG [RS_LOG_REPLAY_OPS-regionserver/fc3fda29f660:0-0 {event_type=RS_LOG_REPLAY, pid=63}] wal.WALSplitter(352): Last flushed sequenceid for 7352dcdbf9811dcebc51a714a1de4f55: last_flushed_sequence_id: 18446744073709551615 2024-11-16T15:00:13,708 DEBUG [RS_LOG_REPLAY_OPS-regionserver/fc3fda29f660:0-0 {event_type=RS_LOG_REPLAY, pid=63}] wal.WALSplitter(352): Last flushed sequenceid for 7f5c9df3a544933227f58b052f743f72: last_flushed_sequence_id: 18446744073709551615 2024-11-16T15:00:13,709 DEBUG [RS_LOG_REPLAY_OPS-regionserver/fc3fda29f660:0-0 {event_type=RS_LOG_REPLAY, pid=63}] wal.WALSplitter(352): Last flushed sequenceid for 4f0d635e77a1e3dd04c4d107bb0f1564: last_flushed_sequence_id: 18446744073709551615 2024-11-16T15:00:13,710 DEBUG [RS_LOG_REPLAY_OPS-regionserver/fc3fda29f660:0-0 {event_type=RS_LOG_REPLAY, pid=63}] wal.WALSplitter(352): Last flushed sequenceid for fd3944fc2b78246a853fb2232c747c35: last_flushed_sequence_id: 18446744073709551615 2024-11-16T15:00:13,711 DEBUG [RS_LOG_REPLAY_OPS-regionserver/fc3fda29f660:0-0 {event_type=RS_LOG_REPLAY, pid=63}] wal.WALSplitter(352): Last flushed sequenceid for 7eb7b75d28b808c5c3f2f96e407b4347: last_flushed_sequence_id: 18446744073709551615 2024-11-16T15:00:13,712 DEBUG [RS_LOG_REPLAY_OPS-regionserver/fc3fda29f660:0-0 {event_type=RS_LOG_REPLAY, pid=63}] wal.WALSplitter(352): Last flushed sequenceid for 426ff231e487ce9f8492776ca1dacc85: last_flushed_sequence_id: 18446744073709551615 2024-11-16T15:00:13,714 DEBUG [RS_LOG_REPLAY_OPS-regionserver/fc3fda29f660:0-0 {event_type=RS_LOG_REPLAY, pid=63}] wal.WALSplitter(352): Last flushed sequenceid for b5566c7e10a0e001c7f58316bdd43c2f: last_flushed_sequence_id: 18446744073709551615 2024-11-16T15:00:13,718 INFO [RS_LOG_REPLAY_OPS-regionserver/fc3fda29f660:0-0-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000004-fc3fda29f660%2C38897%2C1731769023879.1731769025434.temp 2024-11-16T15:00:13,720 INFO [RS_LOG_REPLAY_OPS-regionserver/fc3fda29f660:0-0-Writer-2 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000004-fc3fda29f660%2C38897%2C1731769023879.1731769025434.temp 2024-11-16T15:00:13,721 INFO [RS_LOG_REPLAY_OPS-regionserver/fc3fda29f660:0-0-Writer-1 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000004-fc3fda29f660%2C38897%2C1731769023879.1731769025434.temp 2024-11-16T15:00:13,727 INFO [RS_LOG_REPLAY_OPS-regionserver/fc3fda29f660:0-0-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:42223/user/jenkins/test-data/a8176cb1-2ebb-b530-b3c7-53c9e53c186e/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/4f0d635e77a1e3dd04c4d107bb0f1564/recovered.edits/0000000000000000004-fc3fda29f660%2C38897%2C1731769023879.1731769025434.temp 2024-11-16T15:00:13,727 INFO [RS_LOG_REPLAY_OPS-regionserver/fc3fda29f660:0-0-Writer-2 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:42223/user/jenkins/test-data/a8176cb1-2ebb-b530-b3c7-53c9e53c186e/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/fd3944fc2b78246a853fb2232c747c35/recovered.edits/0000000000000000004-fc3fda29f660%2C38897%2C1731769023879.1731769025434.temp 2024-11-16T15:00:13,727 INFO [RS_LOG_REPLAY_OPS-regionserver/fc3fda29f660:0-0-Writer-1 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:42223/user/jenkins/test-data/a8176cb1-2ebb-b530-b3c7-53c9e53c186e/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/7352dcdbf9811dcebc51a714a1de4f55/recovered.edits/0000000000000000004-fc3fda29f660%2C38897%2C1731769023879.1731769025434.temp 2024-11-16T15:00:13,729 DEBUG [RS_LOG_REPLAY_OPS-regionserver/fc3fda29f660:0-0 {event_type=RS_LOG_REPLAY, pid=63}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:42223/user/jenkins/test-data/a8176cb1-2ebb-b530-b3c7-53c9e53c186e/WALs/fc3fda29f660,38897,1731769023879-splitting/fc3fda29f660%2C38897%2C1731769023879.1731769025434 so closing down 2024-11-16T15:00:13,730 DEBUG [RS_LOG_REPLAY_OPS-regionserver/fc3fda29f660:0-0 {event_type=RS_LOG_REPLAY, pid=63}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-16T15:00:13,731 INFO [RS_LOG_REPLAY_OPS-regionserver/fc3fda29f660:0-0-Writer-2 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000004-fc3fda29f660%2C38897%2C1731769023879.1731769025434.temp 2024-11-16T15:00:13,731 INFO [RS_LOG_REPLAY_OPS-regionserver/fc3fda29f660:0-0-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000004-fc3fda29f660%2C38897%2C1731769023879.1731769025434.temp 2024-11-16T15:00:13,731 INFO [RS_LOG_REPLAY_OPS-regionserver/fc3fda29f660:0-0-Writer-1 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000004-fc3fda29f660%2C38897%2C1731769023879.1731769025434.temp 2024-11-16T15:00:13,735 INFO [RS_LOG_REPLAY_OPS-regionserver/fc3fda29f660:0-0-Writer-2 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:42223/user/jenkins/test-data/a8176cb1-2ebb-b530-b3c7-53c9e53c186e/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/33054df03c1be51dc860405880be6075/recovered.edits/0000000000000000004-fc3fda29f660%2C38897%2C1731769023879.1731769025434.temp 2024-11-16T15:00:13,736 INFO [RS_LOG_REPLAY_OPS-regionserver/fc3fda29f660:0-0-Writer-1 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:42223/user/jenkins/test-data/a8176cb1-2ebb-b530-b3c7-53c9e53c186e/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/7eb7b75d28b808c5c3f2f96e407b4347/recovered.edits/0000000000000000004-fc3fda29f660%2C38897%2C1731769023879.1731769025434.temp 2024-11-16T15:00:13,738 INFO [RS_LOG_REPLAY_OPS-regionserver/fc3fda29f660:0-0-Writer-2 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000004-fc3fda29f660%2C38897%2C1731769023879.1731769025434.temp 2024-11-16T15:00:13,738 INFO [RS_LOG_REPLAY_OPS-regionserver/fc3fda29f660:0-0-Writer-1 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000004-fc3fda29f660%2C38897%2C1731769023879.1731769025434.temp 2024-11-16T15:00:13,743 INFO [RS_LOG_REPLAY_OPS-regionserver/fc3fda29f660:0-0-Writer-2 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:42223/user/jenkins/test-data/a8176cb1-2ebb-b530-b3c7-53c9e53c186e/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/426ff231e487ce9f8492776ca1dacc85/recovered.edits/0000000000000000004-fc3fda29f660%2C38897%2C1731769023879.1731769025434.temp 2024-11-16T15:00:13,743 INFO [RS_LOG_REPLAY_OPS-regionserver/fc3fda29f660:0-0-Writer-1 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:42223/user/jenkins/test-data/a8176cb1-2ebb-b530-b3c7-53c9e53c186e/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/b5566c7e10a0e001c7f58316bdd43c2f/recovered.edits/0000000000000000004-fc3fda29f660%2C38897%2C1731769023879.1731769025434.temp 2024-11-16T15:00:13,743 INFO [RS_LOG_REPLAY_OPS-regionserver/fc3fda29f660:0-0-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:42223/user/jenkins/test-data/a8176cb1-2ebb-b530-b3c7-53c9e53c186e/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/7f5c9df3a544933227f58b052f743f72/recovered.edits/0000000000000000004-fc3fda29f660%2C38897%2C1731769023879.1731769025434.temp 2024-11-16T15:00:13,746 INFO [RS_LOG_REPLAY_OPS-regionserver/fc3fda29f660:0-0-Writer-1 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000004-fc3fda29f660%2C38897%2C1731769023879.1731769025434.temp 2024-11-16T15:00:13,751 INFO [RS_LOG_REPLAY_OPS-regionserver/fc3fda29f660:0-0-Writer-1 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:42223/user/jenkins/test-data/a8176cb1-2ebb-b530-b3c7-53c9e53c186e/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/c2acbc05ae7bb1607a5719b701b40383/recovered.edits/0000000000000000004-fc3fda29f660%2C38897%2C1731769023879.1731769025434.temp 2024-11-16T15:00:13,751 INFO [RS_LOG_REPLAY_OPS-regionserver/fc3fda29f660:0-0 {event_type=RS_LOG_REPLAY, pid=63}] wal.OutputSink(145): 3 split writer threads finished 2024-11-16T15:00:13,769 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36531 is added to blk_1073741869_1045 (size=36767) 2024-11-16T15:00:13,769 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40641 is added to blk_1073741871_1047 (size=36767) 2024-11-16T15:00:13,769 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41671 is added to blk_1073741869_1045 (size=36767) 2024-11-16T15:00:13,770 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36531 is added to blk_1073741871_1047 (size=36767) 2024-11-16T15:00:13,770 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40641 is added to blk_1073741869_1045 (size=36767) 2024-11-16T15:00:13,770 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41671 is added to blk_1073741871_1047 (size=36767) 2024-11-16T15:00:13,770 INFO [split-log-closeStream-pool-1 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:42223/user/jenkins/test-data/a8176cb1-2ebb-b530-b3c7-53c9e53c186e/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/4f0d635e77a1e3dd04c4d107bb0f1564/recovered.edits/0000000000000000004-fc3fda29f660%2C38897%2C1731769023879.1731769025434.temp (wrote 1 edits, skipped 0 edits in 2 ms) 2024-11-16T15:00:13,770 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:42223/user/jenkins/test-data/a8176cb1-2ebb-b530-b3c7-53c9e53c186e/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/b5566c7e10a0e001c7f58316bdd43c2f/recovered.edits/0000000000000000004-fc3fda29f660%2C38897%2C1731769023879.1731769025434.temp (wrote 1 edits, skipped 0 edits in 1 ms) 2024-11-16T15:00:13,772 INFO [split-log-closeStream-pool-1 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:42223/user/jenkins/test-data/a8176cb1-2ebb-b530-b3c7-53c9e53c186e/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/4f0d635e77a1e3dd04c4d107bb0f1564/recovered.edits/0000000000000000004-fc3fda29f660%2C38897%2C1731769023879.1731769025434.temp to hdfs://localhost:42223/user/jenkins/test-data/a8176cb1-2ebb-b530-b3c7-53c9e53c186e/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/4f0d635e77a1e3dd04c4d107bb0f1564/recovered.edits/0000000000000000004 2024-11-16T15:00:13,772 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:42223/user/jenkins/test-data/a8176cb1-2ebb-b530-b3c7-53c9e53c186e/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/b5566c7e10a0e001c7f58316bdd43c2f/recovered.edits/0000000000000000004-fc3fda29f660%2C38897%2C1731769023879.1731769025434.temp to hdfs://localhost:42223/user/jenkins/test-data/a8176cb1-2ebb-b530-b3c7-53c9e53c186e/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/b5566c7e10a0e001c7f58316bdd43c2f/recovered.edits/0000000000000000004 2024-11-16T15:00:13,781 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40641 is added to blk_1073741870_1046 (size=36767) 2024-11-16T15:00:13,781 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41671 is added to blk_1073741870_1046 (size=36767) 2024-11-16T15:00:13,782 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36531 is added to blk_1073741870_1046 (size=36767) 2024-11-16T15:00:13,784 INFO [split-log-closeStream-pool-2 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:42223/user/jenkins/test-data/a8176cb1-2ebb-b530-b3c7-53c9e53c186e/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/426ff231e487ce9f8492776ca1dacc85/recovered.edits/0000000000000000004-fc3fda29f660%2C38897%2C1731769023879.1731769025434.temp (wrote 1 edits, skipped 0 edits in 1 ms) 2024-11-16T15:00:13,786 INFO [split-log-closeStream-pool-2 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:42223/user/jenkins/test-data/a8176cb1-2ebb-b530-b3c7-53c9e53c186e/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/426ff231e487ce9f8492776ca1dacc85/recovered.edits/0000000000000000004-fc3fda29f660%2C38897%2C1731769023879.1731769025434.temp to hdfs://localhost:42223/user/jenkins/test-data/a8176cb1-2ebb-b530-b3c7-53c9e53c186e/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/426ff231e487ce9f8492776ca1dacc85/recovered.edits/0000000000000000004 2024-11-16T15:00:13,800 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36531 is added to blk_1073741872_1048 (size=36767) 2024-11-16T15:00:13,800 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40641 is added to blk_1073741872_1048 (size=36767) 2024-11-16T15:00:13,801 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41671 is added to blk_1073741872_1048 (size=36767) 2024-11-16T15:00:13,801 INFO [split-log-closeStream-pool-1 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:42223/user/jenkins/test-data/a8176cb1-2ebb-b530-b3c7-53c9e53c186e/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/fd3944fc2b78246a853fb2232c747c35/recovered.edits/0000000000000000004-fc3fda29f660%2C38897%2C1731769023879.1731769025434.temp (wrote 1 edits, skipped 0 edits in 1 ms) 2024-11-16T15:00:13,803 INFO [split-log-closeStream-pool-1 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:42223/user/jenkins/test-data/a8176cb1-2ebb-b530-b3c7-53c9e53c186e/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/fd3944fc2b78246a853fb2232c747c35/recovered.edits/0000000000000000004-fc3fda29f660%2C38897%2C1731769023879.1731769025434.temp to hdfs://localhost:42223/user/jenkins/test-data/a8176cb1-2ebb-b530-b3c7-53c9e53c186e/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/fd3944fc2b78246a853fb2232c747c35/recovered.edits/0000000000000000004 2024-11-16T15:00:13,812 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36531 is added to blk_1073741873_1049 (size=36767) 2024-11-16T15:00:13,812 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41671 is added to blk_1073741873_1049 (size=36767) 2024-11-16T15:00:13,813 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40641 is added to blk_1073741873_1049 (size=36767) 2024-11-16T15:00:13,813 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:42223/user/jenkins/test-data/a8176cb1-2ebb-b530-b3c7-53c9e53c186e/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/7f5c9df3a544933227f58b052f743f72/recovered.edits/0000000000000000004-fc3fda29f660%2C38897%2C1731769023879.1731769025434.temp (wrote 1 edits, skipped 0 edits in 1 ms) 2024-11-16T15:00:13,815 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:42223/user/jenkins/test-data/a8176cb1-2ebb-b530-b3c7-53c9e53c186e/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/7f5c9df3a544933227f58b052f743f72/recovered.edits/0000000000000000004-fc3fda29f660%2C38897%2C1731769023879.1731769025434.temp to hdfs://localhost:42223/user/jenkins/test-data/a8176cb1-2ebb-b530-b3c7-53c9e53c186e/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/7f5c9df3a544933227f58b052f743f72/recovered.edits/0000000000000000004 2024-11-16T15:00:13,829 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40641 is added to blk_1073741874_1050 (size=36767) 2024-11-16T15:00:13,829 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36531 is added to blk_1073741874_1050 (size=36767) 2024-11-16T15:00:13,830 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40641 is added to blk_1073741875_1051 (size=36767) 2024-11-16T15:00:13,830 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41671 is added to blk_1073741875_1051 (size=36767) 2024-11-16T15:00:13,831 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41671 is added to blk_1073741874_1050 (size=36767) 2024-11-16T15:00:13,831 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36531 is added to blk_1073741875_1051 (size=36767) 2024-11-16T15:00:13,832 INFO [split-log-closeStream-pool-1 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:42223/user/jenkins/test-data/a8176cb1-2ebb-b530-b3c7-53c9e53c186e/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/7352dcdbf9811dcebc51a714a1de4f55/recovered.edits/0000000000000000004-fc3fda29f660%2C38897%2C1731769023879.1731769025434.temp (wrote 1 edits, skipped 0 edits in 1 ms) 2024-11-16T15:00:13,832 INFO [split-log-closeStream-pool-2 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:42223/user/jenkins/test-data/a8176cb1-2ebb-b530-b3c7-53c9e53c186e/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/33054df03c1be51dc860405880be6075/recovered.edits/0000000000000000004-fc3fda29f660%2C38897%2C1731769023879.1731769025434.temp (wrote 1 edits, skipped 0 edits in 0 ms) 2024-11-16T15:00:13,833 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36531 is added to blk_1073741876_1052 (size=36767) 2024-11-16T15:00:13,834 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41671 is added to blk_1073741876_1052 (size=36767) 2024-11-16T15:00:13,834 INFO [split-log-closeStream-pool-1 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:42223/user/jenkins/test-data/a8176cb1-2ebb-b530-b3c7-53c9e53c186e/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/7352dcdbf9811dcebc51a714a1de4f55/recovered.edits/0000000000000000004-fc3fda29f660%2C38897%2C1731769023879.1731769025434.temp to hdfs://localhost:42223/user/jenkins/test-data/a8176cb1-2ebb-b530-b3c7-53c9e53c186e/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/7352dcdbf9811dcebc51a714a1de4f55/recovered.edits/0000000000000000004 2024-11-16T15:00:13,835 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40641 is added to blk_1073741876_1052 (size=36767) 2024-11-16T15:00:13,835 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:42223/user/jenkins/test-data/a8176cb1-2ebb-b530-b3c7-53c9e53c186e/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/c2acbc05ae7bb1607a5719b701b40383/recovered.edits/0000000000000000004-fc3fda29f660%2C38897%2C1731769023879.1731769025434.temp (wrote 1 edits, skipped 0 edits in 0 ms) 2024-11-16T15:00:13,836 INFO [split-log-closeStream-pool-2 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:42223/user/jenkins/test-data/a8176cb1-2ebb-b530-b3c7-53c9e53c186e/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/33054df03c1be51dc860405880be6075/recovered.edits/0000000000000000004-fc3fda29f660%2C38897%2C1731769023879.1731769025434.temp to hdfs://localhost:42223/user/jenkins/test-data/a8176cb1-2ebb-b530-b3c7-53c9e53c186e/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/33054df03c1be51dc860405880be6075/recovered.edits/0000000000000000004 2024-11-16T15:00:13,838 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:42223/user/jenkins/test-data/a8176cb1-2ebb-b530-b3c7-53c9e53c186e/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/c2acbc05ae7bb1607a5719b701b40383/recovered.edits/0000000000000000004-fc3fda29f660%2C38897%2C1731769023879.1731769025434.temp to hdfs://localhost:42223/user/jenkins/test-data/a8176cb1-2ebb-b530-b3c7-53c9e53c186e/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/c2acbc05ae7bb1607a5719b701b40383/recovered.edits/0000000000000000004 2024-11-16T15:00:13,843 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41671 is added to blk_1073741877_1053 (size=36767) 2024-11-16T15:00:13,843 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40641 is added to blk_1073741877_1053 (size=36767) 2024-11-16T15:00:13,844 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36531 is added to blk_1073741877_1053 (size=36767) 2024-11-16T15:00:14,245 INFO [split-log-closeStream-pool-1 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:42223/user/jenkins/test-data/a8176cb1-2ebb-b530-b3c7-53c9e53c186e/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/7eb7b75d28b808c5c3f2f96e407b4347/recovered.edits/0000000000000000004-fc3fda29f660%2C38897%2C1731769023879.1731769025434.temp (wrote 1 edits, skipped 0 edits in 0 ms) 2024-11-16T15:00:14,247 INFO [split-log-closeStream-pool-1 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:42223/user/jenkins/test-data/a8176cb1-2ebb-b530-b3c7-53c9e53c186e/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/7eb7b75d28b808c5c3f2f96e407b4347/recovered.edits/0000000000000000004-fc3fda29f660%2C38897%2C1731769023879.1731769025434.temp to hdfs://localhost:42223/user/jenkins/test-data/a8176cb1-2ebb-b530-b3c7-53c9e53c186e/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/7eb7b75d28b808c5c3f2f96e407b4347/recovered.edits/0000000000000000004 2024-11-16T15:00:14,247 INFO [RS_LOG_REPLAY_OPS-regionserver/fc3fda29f660:0-0 {event_type=RS_LOG_REPLAY, pid=63}] wal.WALSplitter(425): Processed 18 edits across 9 Regions in 546 ms; skipped=9; WAL=hdfs://localhost:42223/user/jenkins/test-data/a8176cb1-2ebb-b530-b3c7-53c9e53c186e/WALs/fc3fda29f660,38897,1731769023879-splitting/fc3fda29f660%2C38897%2C1731769023879.1731769025434, size=327.1 K, length=334915, corrupted=false, cancelled=false 2024-11-16T15:00:14,247 DEBUG [RS_LOG_REPLAY_OPS-regionserver/fc3fda29f660:0-0 {event_type=RS_LOG_REPLAY, pid=63}] wal.WALSplitter(428): Completed split of hdfs://localhost:42223/user/jenkins/test-data/a8176cb1-2ebb-b530-b3c7-53c9e53c186e/WALs/fc3fda29f660,38897,1731769023879-splitting/fc3fda29f660%2C38897%2C1731769023879.1731769025434, journal: Splitting hdfs://localhost:42223/user/jenkins/test-data/a8176cb1-2ebb-b530-b3c7-53c9e53c186e/WALs/fc3fda29f660,38897,1731769023879-splitting/fc3fda29f660%2C38897%2C1731769023879.1731769025434, size=327.1 K (334915bytes) at 1731769213695Creating recovered edits writer path=hdfs://localhost:42223/user/jenkins/test-data/a8176cb1-2ebb-b530-b3c7-53c9e53c186e/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/4f0d635e77a1e3dd04c4d107bb0f1564/recovered.edits/0000000000000000004-fc3fda29f660%2C38897%2C1731769023879.1731769025434.temp at 1731769213727 (+32 ms)Creating recovered edits writer path=hdfs://localhost:42223/user/jenkins/test-data/a8176cb1-2ebb-b530-b3c7-53c9e53c186e/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/4f0d635e77a1e3dd04c4d107bb0f1564/recovered.edits/0000000000000000004-fc3fda29f660%2C38897%2C1731769023879.1731769025434.temp at 1731769213727Creating recovered edits writer path=hdfs://localhost:42223/user/jenkins/test-data/a8176cb1-2ebb-b530-b3c7-53c9e53c186e/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/7352dcdbf9811dcebc51a714a1de4f55/recovered.edits/0000000000000000004-fc3fda29f660%2C38897%2C1731769023879.1731769025434.temp at 1731769213727Finishing writing output for hdfs://localhost:42223/user/jenkins/test-data/a8176cb1-2ebb-b530-b3c7-53c9e53c186e/WALs/fc3fda29f660,38897,1731769023879-splitting/fc3fda29f660%2C38897%2C1731769023879.1731769025434 so closing down at 1731769213729 (+2 ms)Creating recovered edits writer path=hdfs://localhost:42223/user/jenkins/test-data/a8176cb1-2ebb-b530-b3c7-53c9e53c186e/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/33054df03c1be51dc860405880be6075/recovered.edits/0000000000000000004-fc3fda29f660%2C38897%2C1731769023879.1731769025434.temp at 1731769213735 (+6 ms)Creating recovered edits writer path=hdfs://localhost:42223/user/jenkins/test-data/a8176cb1-2ebb-b530-b3c7-53c9e53c186e/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/7eb7b75d28b808c5c3f2f96e407b4347/recovered.edits/0000000000000000004-fc3fda29f660%2C38897%2C1731769023879.1731769025434.temp at 1731769213736 (+1 ms)Creating recovered edits writer path=hdfs://localhost:42223/user/jenkins/test-data/a8176cb1-2ebb-b530-b3c7-53c9e53c186e/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/426ff231e487ce9f8492776ca1dacc85/recovered.edits/0000000000000000004-fc3fda29f660%2C38897%2C1731769023879.1731769025434.temp at 1731769213743 (+7 ms)Creating recovered edits writer path=hdfs://localhost:42223/user/jenkins/test-data/a8176cb1-2ebb-b530-b3c7-53c9e53c186e/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/b5566c7e10a0e001c7f58316bdd43c2f/recovered.edits/0000000000000000004-fc3fda29f660%2C38897%2C1731769023879.1731769025434.temp at 1731769213743Creating recovered edits writer path=hdfs://localhost:42223/user/jenkins/test-data/a8176cb1-2ebb-b530-b3c7-53c9e53c186e/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/7f5c9df3a544933227f58b052f743f72/recovered.edits/0000000000000000004-fc3fda29f660%2C38897%2C1731769023879.1731769025434.temp at 1731769213743Creating recovered edits writer path=hdfs://localhost:42223/user/jenkins/test-data/a8176cb1-2ebb-b530-b3c7-53c9e53c186e/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/c2acbc05ae7bb1607a5719b701b40383/recovered.edits/0000000000000000004-fc3fda29f660%2C38897%2C1731769023879.1731769025434.temp at 1731769213751 (+8 ms)3 split writer threads finished at 1731769213751Closed recovered edits writer path=hdfs://localhost:42223/user/jenkins/test-data/a8176cb1-2ebb-b530-b3c7-53c9e53c186e/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/4f0d635e77a1e3dd04c4d107bb0f1564/recovered.edits/0000000000000000004-fc3fda29f660%2C38897%2C1731769023879.1731769025434.temp (wrote 1 edits, skipped 0 edits in 2 ms) at 1731769213770 (+19 ms)Closed recovered edits writer path=hdfs://localhost:42223/user/jenkins/test-data/a8176cb1-2ebb-b530-b3c7-53c9e53c186e/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/b5566c7e10a0e001c7f58316bdd43c2f/recovered.edits/0000000000000000004-fc3fda29f660%2C38897%2C1731769023879.1731769025434.temp (wrote 1 edits, skipped 0 edits in 1 ms) at 1731769213770Rename recovered edits hdfs://localhost:42223/user/jenkins/test-data/a8176cb1-2ebb-b530-b3c7-53c9e53c186e/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/4f0d635e77a1e3dd04c4d107bb0f1564/recovered.edits/0000000000000000004-fc3fda29f660%2C38897%2C1731769023879.1731769025434.temp to hdfs://localhost:42223/user/jenkins/test-data/a8176cb1-2ebb-b530-b3c7-53c9e53c186e/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/4f0d635e77a1e3dd04c4d107bb0f1564/recovered.edits/0000000000000000004 at 1731769213772 (+2 ms)Rename recovered edits hdfs://localhost:42223/user/jenkins/test-data/a8176cb1-2ebb-b530-b3c7-53c9e53c186e/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/b5566c7e10a0e001c7f58316bdd43c2f/recovered.edits/0000000000000000004-fc3fda29f660%2C38897%2C1731769023879.1731769025434.temp to hdfs://localhost:42223/user/jenkins/test-data/a8176cb1-2ebb-b530-b3c7-53c9e53c186e/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/b5566c7e10a0e001c7f58316bdd43c2f/recovered.edits/0000000000000000004 at 1731769213772Closed recovered edits writer path=hdfs://localhost:42223/user/jenkins/test-data/a8176cb1-2ebb-b530-b3c7-53c9e53c186e/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/426ff231e487ce9f8492776ca1dacc85/recovered.edits/0000000000000000004-fc3fda29f660%2C38897%2C1731769023879.1731769025434.temp (wrote 1 edits, skipped 0 edits in 1 ms) at 1731769213784 (+12 ms)Rename recovered edits hdfs://localhost:42223/user/jenkins/test-data/a8176cb1-2ebb-b530-b3c7-53c9e53c186e/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/426ff231e487ce9f8492776ca1dacc85/recovered.edits/0000000000000000004-fc3fda29f660%2C38897%2C1731769023879.1731769025434.temp to hdfs://localhost:42223/user/jenkins/test-data/a8176cb1-2ebb-b530-b3c7-53c9e53c186e/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/426ff231e487ce9f8492776ca1dacc85/recovered.edits/0000000000000000004 at 1731769213786 (+2 ms)Closed recovered edits writer path=hdfs://localhost:42223/user/jenkins/test-data/a8176cb1-2ebb-b530-b3c7-53c9e53c186e/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/fd3944fc2b78246a853fb2232c747c35/recovered.edits/0000000000000000004-fc3fda29f660%2C38897%2C1731769023879.1731769025434.temp (wrote 1 edits, skipped 0 edits in 1 ms) at 1731769213801 (+15 ms)Rename recovered edits hdfs://localhost:42223/user/jenkins/test-data/a8176cb1-2ebb-b530-b3c7-53c9e53c186e/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/fd3944fc2b78246a853fb2232c747c35/recovered.edits/0000000000000000004-fc3fda29f660%2C38897%2C1731769023879.1731769025434.temp to hdfs://localhost:42223/user/jenkins/test-data/a8176cb1-2ebb-b530-b3c7-53c9e53c186e/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/fd3944fc2b78246a853fb2232c747c35/recovered.edits/0000000000000000004 at 1731769213803 (+2 ms)Closed recovered edits writer path=hdfs://localhost:42223/user/jenkins/test-data/a8176cb1-2ebb-b530-b3c7-53c9e53c186e/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/7f5c9df3a544933227f58b052f743f72/recovered.edits/0000000000000000004-fc3fda29f660%2C38897%2C1731769023879.1731769025434.temp (wrote 1 edits, skipped 0 edits in 1 ms) at 1731769213813 (+10 ms)Rename recovered edits hdfs://localhost:42223/user/jenkins/test-data/a8176cb1-2ebb-b530-b3c7-53c9e53c186e/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/7f5c9df3a544933227f58b052f743f72/recovered.edits/0000000000000000004-fc3fda29f660%2C38897%2C1731769023879.1731769025434.temp to hdfs://localhost:42223/user/jenkins/test-data/a8176cb1-2ebb-b530-b3c7-53c9e53c186e/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/7f5c9df3a544933227f58b052f743f72/recovered.edits/0000000000000000004 at 1731769213815 (+2 ms)Closed recovered edits writer path=hdfs://localhost:42223/user/jenkins/test-data/a8176cb1-2ebb-b530-b3c7-53c9e53c186e/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/7352dcdbf9811dcebc51a714a1de4f55/recovered.edits/0000000000000000004-fc3fda29f660%2C38897%2C1731769023879.1731769025434.temp (wrote 1 edits, skipped 0 edits in 1 ms) at 1731769213832 (+17 ms)Closed recovered edits writer path=hdfs://localhost:42223/user/jenkins/test-data/a8176cb1-2ebb-b530-b3c7-53c9e53c186e/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/33054df03c1be51dc860405880be6075/recovered.edits/0000000000000000004-fc3fda29f660%2C38897%2C1731769023879.1731769025434.temp (wrote 1 edits, skipped 0 edits in 0 ms) at 1731769213832Rename recovered edits hdfs://localhost:42223/user/jenkins/test-data/a8176cb1-2ebb-b530-b3c7-53c9e53c186e/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/7352dcdbf9811dcebc51a714a1de4f55/recovered.edits/0000000000000000004-fc3fda29f660%2C38897%2C1731769023879.1731769025434.temp to hdfs://localhost:42223/user/jenkins/test-data/a8176cb1-2ebb-b530-b3c7-53c9e53c186e/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/7352dcdbf9811dcebc51a714a1de4f55/recovered.edits/0000000000000000004 at 1731769213834 (+2 ms)Closed recovered edits writer path=hdfs://localhost:42223/user/jenkins/test-data/a8176cb1-2ebb-b530-b3c7-53c9e53c186e/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/c2acbc05ae7bb1607a5719b701b40383/recovered.edits/0000000000000000004-fc3fda29f660%2C38897%2C1731769023879.1731769025434.temp (wrote 1 edits, skipped 0 edits in 0 ms) at 1731769213835 (+1 ms)Rename recovered edits hdfs://localhost:42223/user/jenkins/test-data/a8176cb1-2ebb-b530-b3c7-53c9e53c186e/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/33054df03c1be51dc860405880be6075/recovered.edits/0000000000000000004-fc3fda29f660%2C38897%2C1731769023879.1731769025434.temp to hdfs://localhost:42223/user/jenkins/test-data/a8176cb1-2ebb-b530-b3c7-53c9e53c186e/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/33054df03c1be51dc860405880be6075/recovered.edits/0000000000000000004 at 1731769213836 (+1 ms)Rename recovered edits hdfs://localhost:42223/user/jenkins/test-data/a8176cb1-2ebb-b530-b3c7-53c9e53c186e/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/c2acbc05ae7bb1607a5719b701b40383/recovered.edits/0000000000000000004-fc3fda29f660%2C38897%2C1731769023879.1731769025434.temp to hdfs://localhost:42223/user/jenkins/test-data/a8176cb1-2ebb-b530-b3c7-53c9e53c186e/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/c2acbc05ae7bb1607a5719b701b40383/recovered.edits/0000000000000000004 at 1731769213838 (+2 ms)Closed recovered edits writer path=hdfs://localhost:42223/user/jenkins/test-data/a8176cb1-2ebb-b530-b3c7-53c9e53c186e/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/7eb7b75d28b808c5c3f2f96e407b4347/recovered.edits/0000000000000000004-fc3fda29f660%2C38897%2C1731769023879.1731769025434.temp (wrote 1 edits, skipped 0 edits in 0 ms) at 1731769214245 (+407 ms)Rename recovered edits hdfs://localhost:42223/user/jenkins/test-data/a8176cb1-2ebb-b530-b3c7-53c9e53c186e/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/7eb7b75d28b808c5c3f2f96e407b4347/recovered.edits/0000000000000000004-fc3fda29f660%2C38897%2C1731769023879.1731769025434.temp to hdfs://localhost:42223/user/jenkins/test-data/a8176cb1-2ebb-b530-b3c7-53c9e53c186e/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/7eb7b75d28b808c5c3f2f96e407b4347/recovered.edits/0000000000000000004 at 1731769214247 (+2 ms)Processed 18 edits across 9 Regions in 546 ms; skipped=9; WAL=hdfs://localhost:42223/user/jenkins/test-data/a8176cb1-2ebb-b530-b3c7-53c9e53c186e/WALs/fc3fda29f660,38897,1731769023879-splitting/fc3fda29f660%2C38897%2C1731769023879.1731769025434, size=327.1 K, length=334915, corrupted=false, cancelled=false at 1731769214247 2024-11-16T15:00:14,248 DEBUG [RS_LOG_REPLAY_OPS-regionserver/fc3fda29f660:0-0 {event_type=RS_LOG_REPLAY, pid=63}] regionserver.SplitLogWorker(218): Done splitting WAL hdfs://localhost:42223/user/jenkins/test-data/a8176cb1-2ebb-b530-b3c7-53c9e53c186e/WALs/fc3fda29f660,38897,1731769023879-splitting/fc3fda29f660%2C38897%2C1731769023879.1731769025434 2024-11-16T15:00:14,248 DEBUG [RS_LOG_REPLAY_OPS-regionserver/fc3fda29f660:0-0 {event_type=RS_LOG_REPLAY, pid=63}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=63 2024-11-16T15:00:14,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43615 {}] master.HMaster(4169): Remote procedure done, pid=63 2024-11-16T15:00:14,253 INFO [PEWorker-1 {}] wal.WALSplitUtil(143): Moved hdfs://localhost:42223/user/jenkins/test-data/a8176cb1-2ebb-b530-b3c7-53c9e53c186e/WALs/fc3fda29f660,38897,1731769023879-splitting/fc3fda29f660%2C38897%2C1731769023879.1731769025434 to hdfs://localhost:42223/user/jenkins/test-data/a8176cb1-2ebb-b530-b3c7-53c9e53c186e/oldWALs 2024-11-16T15:00:14,256 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=63, resume processing ppid=62 2024-11-16T15:00:14,256 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure fc3fda29f660%2C38897%2C1731769023879.1731769025434, worker=fc3fda29f660,39089,1731769023769 in 727 msec 2024-11-16T15:00:14,257 DEBUG [PEWorker-1 {}] master.SplitWALManager(172): Release split WAL worker=fc3fda29f660,39089,1731769023769 2024-11-16T15:00:14,260 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=62, resume processing ppid=57 2024-11-16T15:00:14,260 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure fc3fda29f660%2C38897%2C1731769023879.1731769025434, worker=fc3fda29f660,39089,1731769023769 in 733 msec 2024-11-16T15:00:14,262 INFO [PEWorker-1 {}] master.SplitLogManager(171): hdfs://localhost:42223/user/jenkins/test-data/a8176cb1-2ebb-b530-b3c7-53c9e53c186e/WALs/fc3fda29f660,38897,1731769023879-splitting dir is empty, no logs to split. 2024-11-16T15:00:14,262 INFO [PEWorker-1 {}] master.SplitWALManager(105): fc3fda29f660,38897,1731769023879 WAL count=0, meta=false 2024-11-16T15:00:14,262 DEBUG [PEWorker-1 {}] procedure.ServerCrashProcedure(329): Check if fc3fda29f660,38897,1731769023879 WAL splitting is done? wals=0, meta=false 2024-11-16T15:00:14,263 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(132): Toggle KILL before store update to: false 2024-11-16T15:00:14,263 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update: pid=57, state=RUNNABLE:SERVER_CRASH_ASSIGN, hasLock=true; ServerCrashProcedure fc3fda29f660,38897,1731769023879, splitWal=true, meta=true 2024-11-16T15:00:14,263 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-16T15:00:14,263 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update: pid=57, state=RUNNABLE:SERVER_CRASH_ASSIGN, hasLock=true; ServerCrashProcedure fc3fda29f660,38897,1731769023879, splitWal=true, meta=true at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1879) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T15:00:14,271 INFO [Time-limited test {}] procedure.MasterProcedureTestingUtility(473): Restart 6 exec state=pid=57, state=RUNNABLE:SERVER_CRASH_ASSIGN, hasLock=true; ServerCrashProcedure fc3fda29f660,38897,1731769023879, splitWal=true, meta=true 2024-11-16T15:00:14,272 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-16T15:00:14,272 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-16T15:00:14,272 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-16T15:00:14,272 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-16T15:00:14,272 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-16T15:00:14,272 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-16T15:00:14,272 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-16T15:00:14,273 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-16T15:00:14,273 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-16T15:00:14,276 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=57, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-16T15:00:14,276 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=58, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-16T15:00:14,276 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=59, ppid=58, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-16T15:00:14,276 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-16T15:00:14,276 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-16T15:00:14,276 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-16T15:00:14,276 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=62, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-16T15:00:14,276 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-16T15:00:14,277 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true 2024-11-16T15:00:14,278 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=57, state=RUNNABLE:SERVER_CRASH_DELETE_SPLIT_WALS_DIR, hasLock=false; ServerCrashProcedure fc3fda29f660,38897,1731769023879, splitWal=true, meta=true 2024-11-16T15:00:14,278 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure fc3fda29f660%2C38897%2C1731769023879.meta.1731769025887.meta, worker=fc3fda29f660,42985,1731769023951 2024-11-16T15:00:14,278 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure fc3fda29f660%2C38897%2C1731769023879.meta.1731769025887.meta, worker=fc3fda29f660,42985,1731769023951 2024-11-16T15:00:14,278 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-16T15:00:14,278 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=fc3fda29f660,39089,1731769023769 2024-11-16T15:00:14,278 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure fc3fda29f660%2C38897%2C1731769023879.1731769025434, worker=fc3fda29f660,39089,1731769023769 2024-11-16T15:00:14,278 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure fc3fda29f660%2C38897%2C1731769023879.1731769025434, worker=fc3fda29f660,39089,1731769023769 2024-11-16T15:00:14,278 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=57, state=RUNNABLE:SERVER_CRASH_DELETE_SPLIT_WALS_DIR, hasLock=false; ServerCrashProcedure fc3fda29f660,38897,1731769023879, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-16T15:00:14,278 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure fc3fda29f660%2C38897%2C1731769023879.meta.1731769025887.meta, worker=fc3fda29f660,42985,1731769023951 didn't hold the lock before restarting, skip acquiring lock. 2024-11-16T15:00:14,278 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure fc3fda29f660%2C38897%2C1731769023879.meta.1731769025887.meta, worker=fc3fda29f660,42985,1731769023951 didn't hold the lock before restarting, skip acquiring lock. 2024-11-16T15:00:14,278 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-16T15:00:14,278 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=fc3fda29f660,39089,1731769023769 didn't hold the lock before restarting, skip acquiring lock. 2024-11-16T15:00:14,278 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure fc3fda29f660%2C38897%2C1731769023879.1731769025434, worker=fc3fda29f660,39089,1731769023769 didn't hold the lock before restarting, skip acquiring lock. 2024-11-16T15:00:14,278 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure fc3fda29f660%2C38897%2C1731769023879.1731769025434, worker=fc3fda29f660,39089,1731769023769 didn't hold the lock before restarting, skip acquiring lock. 2024-11-16T15:00:14,279 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 5 msec 2024-11-16T15:00:14,279 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=fc3fda29f660,39089,1731769023769, regionLocation=fc3fda29f660,39089,1731769023769, openSeqNum=61 2024-11-16T15:00:14,279 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=fc3fda29f660,39089,1731769023769, table=hbase:meta, region=1588230740 2024-11-16T15:00:14,280 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as fc3fda29f660,39089,1731769023769, state=OPEN 2024-11-16T15:00:14,328 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43615-0x101456fc9c70000, quorum=127.0.0.1:63224, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-16T15:00:14,328 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42985-0x101456fc9c70003, quorum=127.0.0.1:63224, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-16T15:00:14,328 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39089-0x101456fc9c70001, quorum=127.0.0.1:63224, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-16T15:00:14,329 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-16T15:00:14,329 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-16T15:00:14,329 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-16T15:00:14,332 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731769244332 2024-11-16T15:00:14,332 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-16T15:00:14,338 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4f0d635e77a1e3dd04c4d107bb0f1564, regionState=OPEN, lastHost=fc3fda29f660,38897,1731769023879, regionLocation=fc3fda29f660,38897,1731769023879, openSeqNum=2 2024-11-16T15:00:14,338 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c3dde5f5f668d914608a2987d4f4c001, regionState=OPEN, lastHost=fc3fda29f660,42985,1731769023951, regionLocation=fc3fda29f660,42985,1731769023951, openSeqNum=2 2024-11-16T15:00:14,339 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5bb22144956dcd43c9b1d957232b11e7, regionState=OPEN, lastHost=fc3fda29f660,39089,1731769023769, regionLocation=fc3fda29f660,39089,1731769023769, openSeqNum=2 2024-11-16T15:00:14,339 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=dde7f54607f11c5fb4271e361e82dabd, regionState=OPEN, lastHost=fc3fda29f660,39089,1731769023769, regionLocation=fc3fda29f660,39089,1731769023769, openSeqNum=2 2024-11-16T15:00:14,339 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=fd3944fc2b78246a853fb2232c747c35, regionState=OPEN, lastHost=fc3fda29f660,38897,1731769023879, regionLocation=fc3fda29f660,38897,1731769023879, openSeqNum=2 2024-11-16T15:00:14,339 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a5db9431e22152bd2a82e84504d2879a, regionState=OPEN, lastHost=fc3fda29f660,42985,1731769023951, regionLocation=fc3fda29f660,42985,1731769023951, openSeqNum=2 2024-11-16T15:00:14,339 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=dcd6414ddebad66e87ab37a6dba924f3, regionState=OPEN, lastHost=fc3fda29f660,42985,1731769023951, regionLocation=fc3fda29f660,42985,1731769023951, openSeqNum=2 2024-11-16T15:00:14,339 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1d97df67e5110b0fe1a1e4b7c1341cbb, regionState=OPEN, lastHost=fc3fda29f660,42985,1731769023951, regionLocation=fc3fda29f660,42985,1731769023951, openSeqNum=2 2024-11-16T15:00:14,339 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7352dcdbf9811dcebc51a714a1de4f55, regionState=OPEN, lastHost=fc3fda29f660,38897,1731769023879, regionLocation=fc3fda29f660,38897,1731769023879, openSeqNum=2 2024-11-16T15:00:14,340 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7eb7b75d28b808c5c3f2f96e407b4347, regionState=OPEN, lastHost=fc3fda29f660,38897,1731769023879, regionLocation=fc3fda29f660,38897,1731769023879, openSeqNum=2 2024-11-16T15:00:14,340 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b4f73326a2a6a279d503e557c6ed452f, regionState=OPEN, lastHost=fc3fda29f660,39089,1731769023769, regionLocation=fc3fda29f660,39089,1731769023769, openSeqNum=2 2024-11-16T15:00:14,340 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=266565e60d7419a2a30ceec1b0f6519b, regionState=OPEN, lastHost=fc3fda29f660,42985,1731769023951, regionLocation=fc3fda29f660,42985,1731769023951, openSeqNum=2 2024-11-16T15:00:14,340 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c2acbc05ae7bb1607a5719b701b40383, regionState=OPEN, lastHost=fc3fda29f660,38897,1731769023879, regionLocation=fc3fda29f660,38897,1731769023879, openSeqNum=2 2024-11-16T15:00:14,340 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=994ca1526cf6c4838e65f85444853f1a, regionState=OPEN, lastHost=fc3fda29f660,39089,1731769023769, regionLocation=fc3fda29f660,39089,1731769023769, openSeqNum=2 2024-11-16T15:00:14,340 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=299131947f15f9c44bcfe22cd6623096, regionState=OPEN, lastHost=fc3fda29f660,42985,1731769023951, regionLocation=fc3fda29f660,42985,1731769023951, openSeqNum=2 2024-11-16T15:00:14,340 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7f5c9df3a544933227f58b052f743f72, regionState=OPEN, lastHost=fc3fda29f660,38897,1731769023879, regionLocation=fc3fda29f660,38897,1731769023879, openSeqNum=2 2024-11-16T15:00:14,340 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=18f03e2509a3a89e4a69ba87fad73940, regionState=OPEN, lastHost=fc3fda29f660,39089,1731769023769, regionLocation=fc3fda29f660,39089,1731769023769, openSeqNum=2 2024-11-16T15:00:14,340 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b364a006589fe0d2474ffa45692a6efc, regionState=OPEN, lastHost=fc3fda29f660,42985,1731769023951, regionLocation=fc3fda29f660,42985,1731769023951, openSeqNum=2 2024-11-16T15:00:14,341 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=33054df03c1be51dc860405880be6075, regionState=OPEN, lastHost=fc3fda29f660,38897,1731769023879, regionLocation=fc3fda29f660,38897,1731769023879, openSeqNum=2 2024-11-16T15:00:14,341 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7f11f79823720d664105d8643dfa862d, regionState=OPEN, lastHost=fc3fda29f660,39089,1731769023769, regionLocation=fc3fda29f660,39089,1731769023769, openSeqNum=2 2024-11-16T15:00:14,341 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=35ef9a7f35bb0b08ec1d53b82d3f9777, regionState=OPEN, lastHost=fc3fda29f660,42985,1731769023951, regionLocation=fc3fda29f660,42985,1731769023951, openSeqNum=2 2024-11-16T15:00:14,341 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b5566c7e10a0e001c7f58316bdd43c2f, regionState=OPEN, lastHost=fc3fda29f660,38897,1731769023879, regionLocation=fc3fda29f660,38897,1731769023879, openSeqNum=2 2024-11-16T15:00:14,341 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=62a6d124ad9d161782c0632102c766de, regionState=OPEN, lastHost=fc3fda29f660,39089,1731769023769, regionLocation=fc3fda29f660,39089,1731769023769, openSeqNum=2 2024-11-16T15:00:14,341 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=dc4fcaece350cd66f6bf88cfbdff6f15, regionState=OPEN, lastHost=fc3fda29f660,42985,1731769023951, regionLocation=fc3fda29f660,42985,1731769023951, openSeqNum=2 2024-11-16T15:00:14,341 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=426ff231e487ce9f8492776ca1dacc85, regionState=OPEN, lastHost=fc3fda29f660,38897,1731769023879, regionLocation=fc3fda29f660,38897,1731769023879, openSeqNum=2 2024-11-16T15:00:14,341 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2f41a8c996224c818e917389bd3f5141, regionState=OPEN, lastHost=fc3fda29f660,39089,1731769023769, regionLocation=fc3fda29f660,39089,1731769023769, openSeqNum=2 2024-11-16T15:00:14,341 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-11-16T15:00:14,341 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731769274341 2024-11-16T15:00:14,341 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731769334341 2024-11-16T15:00:14,341 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 8 msec 2024-11-16T15:00:14,343 INFO [PEWorker-1 {}] master.MasterWalManager(331): Log dir for server fc3fda29f660,38897,1731769023879 does not exist 2024-11-16T15:00:14,343 INFO [PEWorker-1 {}] master.SplitWALManager(105): fc3fda29f660,38897,1731769023879 WAL count=0, meta=false 2024-11-16T15:00:14,343 DEBUG [PEWorker-1 {}] procedure.ServerCrashProcedure(329): Check if fc3fda29f660,38897,1731769023879 WAL splitting is done? wals=0, meta=false 2024-11-16T15:00:14,344 INFO [PEWorker-1 {}] procedure.ServerCrashProcedure(321): Remove WAL directory for fc3fda29f660,38897,1731769023879 failed, ignore...File hdfs://localhost:42223/user/jenkins/test-data/a8176cb1-2ebb-b530-b3c7-53c9e53c186e/WALs/fc3fda29f660,38897,1731769023879-splitting does not exist. 2024-11-16T15:00:14,344 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(132): Toggle KILL before store update to: true 2024-11-16T15:00:14,345 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=7352dcdbf9811dcebc51a714a1de4f55, ASSIGN}, {pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=7f5c9df3a544933227f58b052f743f72, ASSIGN}, {pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=426ff231e487ce9f8492776ca1dacc85, ASSIGN}, {pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=fd3944fc2b78246a853fb2232c747c35, ASSIGN}, {pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=7eb7b75d28b808c5c3f2f96e407b4347, ASSIGN}, {pid=69, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=33054df03c1be51dc860405880be6075, ASSIGN}, {pid=70, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=b5566c7e10a0e001c7f58316bdd43c2f, ASSIGN}, {pid=71, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=4f0d635e77a1e3dd04c4d107bb0f1564, ASSIGN}, {pid=72, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=c2acbc05ae7bb1607a5719b701b40383, ASSIGN}] 2024-11-16T15:00:14,345 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(132): Toggle KILL before store update to: false 2024-11-16T15:00:14,345 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update: pid=57, state=WAITING:SERVER_CRASH_FINISH, hasLock=true; ServerCrashProcedure fc3fda29f660,38897,1731769023879, splitWal=true, meta=true 2024-11-16T15:00:14,345 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-16T15:00:14,345 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update: pid=57, state=WAITING:SERVER_CRASH_FINISH, hasLock=true; ServerCrashProcedure fc3fda29f660,38897,1731769023879, splitWal=true, meta=true at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1879) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T15:00:14,592 INFO [Time-limited test {}] procedure.MasterProcedureTestingUtility(473): Restart 7 exec state=pid=57, state=WAITING:SERVER_CRASH_FINISH, hasLock=true; ServerCrashProcedure fc3fda29f660,38897,1731769023879, splitWal=true, meta=true 2024-11-16T15:00:14,592 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-16T15:00:14,592 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-16T15:00:14,592 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-16T15:00:14,592 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-16T15:00:14,592 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-16T15:00:14,592 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-16T15:00:14,592 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-16T15:00:14,592 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-16T15:00:14,593 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-16T15:00:14,595 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=57, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-16T15:00:14,595 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=58, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-16T15:00:14,595 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=59, ppid=58, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-16T15:00:14,595 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-16T15:00:14,595 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-16T15:00:14,595 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-16T15:00:14,595 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=62, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-16T15:00:14,595 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-16T15:00:14,596 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true 2024-11-16T15:00:14,597 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=57, state=RUNNABLE:SERVER_CRASH_ASSIGN, hasLock=false; ServerCrashProcedure fc3fda29f660,38897,1731769023879, splitWal=true, meta=true 2024-11-16T15:00:14,597 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure fc3fda29f660%2C38897%2C1731769023879.meta.1731769025887.meta, worker=fc3fda29f660,42985,1731769023951 2024-11-16T15:00:14,597 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure fc3fda29f660%2C38897%2C1731769023879.meta.1731769025887.meta, worker=fc3fda29f660,42985,1731769023951 2024-11-16T15:00:14,597 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-16T15:00:14,597 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=fc3fda29f660,39089,1731769023769 2024-11-16T15:00:14,597 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure fc3fda29f660%2C38897%2C1731769023879.1731769025434, worker=fc3fda29f660,39089,1731769023769 2024-11-16T15:00:14,597 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure fc3fda29f660%2C38897%2C1731769023879.1731769025434, worker=fc3fda29f660,39089,1731769023769 2024-11-16T15:00:14,597 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=57, state=RUNNABLE:SERVER_CRASH_ASSIGN, hasLock=false; ServerCrashProcedure fc3fda29f660,38897,1731769023879, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-16T15:00:14,597 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure fc3fda29f660%2C38897%2C1731769023879.meta.1731769025887.meta, worker=fc3fda29f660,42985,1731769023951 didn't hold the lock before restarting, skip acquiring lock. 2024-11-16T15:00:14,597 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure fc3fda29f660%2C38897%2C1731769023879.meta.1731769025887.meta, worker=fc3fda29f660,42985,1731769023951 didn't hold the lock before restarting, skip acquiring lock. 2024-11-16T15:00:14,597 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-16T15:00:14,597 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=fc3fda29f660,39089,1731769023769 didn't hold the lock before restarting, skip acquiring lock. 2024-11-16T15:00:14,597 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure fc3fda29f660%2C38897%2C1731769023879.1731769025434, worker=fc3fda29f660,39089,1731769023769 didn't hold the lock before restarting, skip acquiring lock. 2024-11-16T15:00:14,597 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure fc3fda29f660%2C38897%2C1731769023879.1731769025434, worker=fc3fda29f660,39089,1731769023769 didn't hold the lock before restarting, skip acquiring lock. 2024-11-16T15:00:14,597 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 4 msec 2024-11-16T15:00:14,598 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=fc3fda29f660,39089,1731769023769, regionLocation=fc3fda29f660,39089,1731769023769, openSeqNum=61 2024-11-16T15:00:14,598 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=fc3fda29f660,39089,1731769023769, table=hbase:meta, region=1588230740 2024-11-16T15:00:14,598 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as fc3fda29f660,39089,1731769023769, state=OPEN 2024-11-16T15:00:14,620 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42985-0x101456fc9c70003, quorum=127.0.0.1:63224, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-16T15:00:14,620 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39089-0x101456fc9c70001, quorum=127.0.0.1:63224, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-16T15:00:14,620 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43615-0x101456fc9c70000, quorum=127.0.0.1:63224, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-16T15:00:14,620 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-16T15:00:14,620 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-16T15:00:14,620 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-16T15:00:14,621 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731769244621 2024-11-16T15:00:14,621 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-16T15:00:14,626 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4f0d635e77a1e3dd04c4d107bb0f1564, regionState=OPEN, lastHost=fc3fda29f660,38897,1731769023879, regionLocation=fc3fda29f660,38897,1731769023879, openSeqNum=2 2024-11-16T15:00:14,626 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c3dde5f5f668d914608a2987d4f4c001, regionState=OPEN, lastHost=fc3fda29f660,42985,1731769023951, regionLocation=fc3fda29f660,42985,1731769023951, openSeqNum=2 2024-11-16T15:00:14,626 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5bb22144956dcd43c9b1d957232b11e7, regionState=OPEN, lastHost=fc3fda29f660,39089,1731769023769, regionLocation=fc3fda29f660,39089,1731769023769, openSeqNum=2 2024-11-16T15:00:14,626 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=dde7f54607f11c5fb4271e361e82dabd, regionState=OPEN, lastHost=fc3fda29f660,39089,1731769023769, regionLocation=fc3fda29f660,39089,1731769023769, openSeqNum=2 2024-11-16T15:00:14,626 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=fd3944fc2b78246a853fb2232c747c35, regionState=OPEN, lastHost=fc3fda29f660,38897,1731769023879, regionLocation=fc3fda29f660,38897,1731769023879, openSeqNum=2 2024-11-16T15:00:14,626 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a5db9431e22152bd2a82e84504d2879a, regionState=OPEN, lastHost=fc3fda29f660,42985,1731769023951, regionLocation=fc3fda29f660,42985,1731769023951, openSeqNum=2 2024-11-16T15:00:14,626 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=dcd6414ddebad66e87ab37a6dba924f3, regionState=OPEN, lastHost=fc3fda29f660,42985,1731769023951, regionLocation=fc3fda29f660,42985,1731769023951, openSeqNum=2 2024-11-16T15:00:14,627 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1d97df67e5110b0fe1a1e4b7c1341cbb, regionState=OPEN, lastHost=fc3fda29f660,42985,1731769023951, regionLocation=fc3fda29f660,42985,1731769023951, openSeqNum=2 2024-11-16T15:00:14,627 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7352dcdbf9811dcebc51a714a1de4f55, regionState=OPEN, lastHost=fc3fda29f660,38897,1731769023879, regionLocation=fc3fda29f660,38897,1731769023879, openSeqNum=2 2024-11-16T15:00:14,627 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7eb7b75d28b808c5c3f2f96e407b4347, regionState=OPEN, lastHost=fc3fda29f660,38897,1731769023879, regionLocation=fc3fda29f660,38897,1731769023879, openSeqNum=2 2024-11-16T15:00:14,627 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b4f73326a2a6a279d503e557c6ed452f, regionState=OPEN, lastHost=fc3fda29f660,39089,1731769023769, regionLocation=fc3fda29f660,39089,1731769023769, openSeqNum=2 2024-11-16T15:00:14,627 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=266565e60d7419a2a30ceec1b0f6519b, regionState=OPEN, lastHost=fc3fda29f660,42985,1731769023951, regionLocation=fc3fda29f660,42985,1731769023951, openSeqNum=2 2024-11-16T15:00:14,627 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c2acbc05ae7bb1607a5719b701b40383, regionState=OPEN, lastHost=fc3fda29f660,38897,1731769023879, regionLocation=fc3fda29f660,38897,1731769023879, openSeqNum=2 2024-11-16T15:00:14,627 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=994ca1526cf6c4838e65f85444853f1a, regionState=OPEN, lastHost=fc3fda29f660,39089,1731769023769, regionLocation=fc3fda29f660,39089,1731769023769, openSeqNum=2 2024-11-16T15:00:14,627 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=299131947f15f9c44bcfe22cd6623096, regionState=OPEN, lastHost=fc3fda29f660,42985,1731769023951, regionLocation=fc3fda29f660,42985,1731769023951, openSeqNum=2 2024-11-16T15:00:14,628 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7f5c9df3a544933227f58b052f743f72, regionState=OPEN, lastHost=fc3fda29f660,38897,1731769023879, regionLocation=fc3fda29f660,38897,1731769023879, openSeqNum=2 2024-11-16T15:00:14,628 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=18f03e2509a3a89e4a69ba87fad73940, regionState=OPEN, lastHost=fc3fda29f660,39089,1731769023769, regionLocation=fc3fda29f660,39089,1731769023769, openSeqNum=2 2024-11-16T15:00:14,628 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b364a006589fe0d2474ffa45692a6efc, regionState=OPEN, lastHost=fc3fda29f660,42985,1731769023951, regionLocation=fc3fda29f660,42985,1731769023951, openSeqNum=2 2024-11-16T15:00:14,628 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=33054df03c1be51dc860405880be6075, regionState=OPEN, lastHost=fc3fda29f660,38897,1731769023879, regionLocation=fc3fda29f660,38897,1731769023879, openSeqNum=2 2024-11-16T15:00:14,628 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7f11f79823720d664105d8643dfa862d, regionState=OPEN, lastHost=fc3fda29f660,39089,1731769023769, regionLocation=fc3fda29f660,39089,1731769023769, openSeqNum=2 2024-11-16T15:00:14,628 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=35ef9a7f35bb0b08ec1d53b82d3f9777, regionState=OPEN, lastHost=fc3fda29f660,42985,1731769023951, regionLocation=fc3fda29f660,42985,1731769023951, openSeqNum=2 2024-11-16T15:00:14,628 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b5566c7e10a0e001c7f58316bdd43c2f, regionState=OPEN, lastHost=fc3fda29f660,38897,1731769023879, regionLocation=fc3fda29f660,38897,1731769023879, openSeqNum=2 2024-11-16T15:00:14,628 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=62a6d124ad9d161782c0632102c766de, regionState=OPEN, lastHost=fc3fda29f660,39089,1731769023769, regionLocation=fc3fda29f660,39089,1731769023769, openSeqNum=2 2024-11-16T15:00:14,628 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=dc4fcaece350cd66f6bf88cfbdff6f15, regionState=OPEN, lastHost=fc3fda29f660,42985,1731769023951, regionLocation=fc3fda29f660,42985,1731769023951, openSeqNum=2 2024-11-16T15:00:14,629 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=426ff231e487ce9f8492776ca1dacc85, regionState=OPEN, lastHost=fc3fda29f660,38897,1731769023879, regionLocation=fc3fda29f660,38897,1731769023879, openSeqNum=2 2024-11-16T15:00:14,629 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2f41a8c996224c818e917389bd3f5141, regionState=OPEN, lastHost=fc3fda29f660,39089,1731769023769, regionLocation=fc3fda29f660,39089,1731769023769, openSeqNum=2 2024-11-16T15:00:14,629 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-11-16T15:00:14,629 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731769274629 2024-11-16T15:00:14,629 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731769334629 2024-11-16T15:00:14,629 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 7 msec 2024-11-16T15:00:14,629 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=7352dcdbf9811dcebc51a714a1de4f55, ASSIGN}, {pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=7f5c9df3a544933227f58b052f743f72, ASSIGN}, {pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=426ff231e487ce9f8492776ca1dacc85, ASSIGN}, {pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=fd3944fc2b78246a853fb2232c747c35, ASSIGN}, {pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=7eb7b75d28b808c5c3f2f96e407b4347, ASSIGN}, {pid=69, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=33054df03c1be51dc860405880be6075, ASSIGN}, {pid=70, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=b5566c7e10a0e001c7f58316bdd43c2f, ASSIGN}, {pid=71, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=4f0d635e77a1e3dd04c4d107bb0f1564, ASSIGN}, {pid=72, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=c2acbc05ae7bb1607a5719b701b40383, ASSIGN}] 2024-11-16T15:00:14,630 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(132): Toggle KILL before store update to: true 2024-11-16T15:00:14,632 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=72, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=c2acbc05ae7bb1607a5719b701b40383, ASSIGN 2024-11-16T15:00:14,633 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=72, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=c2acbc05ae7bb1607a5719b701b40383, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-11-16T15:00:14,635 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=71, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=4f0d635e77a1e3dd04c4d107bb0f1564, ASSIGN 2024-11-16T15:00:14,635 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=71, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=4f0d635e77a1e3dd04c4d107bb0f1564, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-11-16T15:00:14,636 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=70, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=b5566c7e10a0e001c7f58316bdd43c2f, ASSIGN 2024-11-16T15:00:14,637 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=70, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=b5566c7e10a0e001c7f58316bdd43c2f, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-11-16T15:00:14,638 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=69, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=33054df03c1be51dc860405880be6075, ASSIGN 2024-11-16T15:00:14,638 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=69, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=33054df03c1be51dc860405880be6075, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-11-16T15:00:14,639 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=7eb7b75d28b808c5c3f2f96e407b4347, ASSIGN 2024-11-16T15:00:14,640 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=7eb7b75d28b808c5c3f2f96e407b4347, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-11-16T15:00:14,641 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=fd3944fc2b78246a853fb2232c747c35, ASSIGN 2024-11-16T15:00:14,642 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=fd3944fc2b78246a853fb2232c747c35, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-11-16T15:00:14,642 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=426ff231e487ce9f8492776ca1dacc85, ASSIGN 2024-11-16T15:00:14,643 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=426ff231e487ce9f8492776ca1dacc85, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-11-16T15:00:14,644 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=7f5c9df3a544933227f58b052f743f72, ASSIGN 2024-11-16T15:00:14,645 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=7f5c9df3a544933227f58b052f743f72, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-11-16T15:00:14,646 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=7352dcdbf9811dcebc51a714a1de4f55, ASSIGN 2024-11-16T15:00:14,647 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=7352dcdbf9811dcebc51a714a1de4f55, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-11-16T15:00:14,785 DEBUG [fc3fda29f660:43615 {}] balancer.BalancerClusterState(204): Hosts are {fc3fda29f660=0} racks are {/default-rack=0} 2024-11-16T15:00:14,785 DEBUG [fc3fda29f660:43615 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-11-16T15:00:14,785 DEBUG [fc3fda29f660:43615 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-11-16T15:00:14,785 DEBUG [fc3fda29f660:43615 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-11-16T15:00:14,785 DEBUG [fc3fda29f660:43615 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-11-16T15:00:14,785 INFO [fc3fda29f660:43615 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-11-16T15:00:14,786 INFO [fc3fda29f660:43615 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-11-16T15:00:14,786 DEBUG [fc3fda29f660:43615 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-11-16T15:00:14,787 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=66 updating hbase:meta row=426ff231e487ce9f8492776ca1dacc85, regionState=OPENING, regionLocation=fc3fda29f660,39089,1731769023769 2024-11-16T15:00:14,788 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=69 updating hbase:meta row=33054df03c1be51dc860405880be6075, regionState=OPENING, regionLocation=fc3fda29f660,39089,1731769023769 2024-11-16T15:00:14,789 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=72 updating hbase:meta row=c2acbc05ae7bb1607a5719b701b40383, regionState=OPENING, regionLocation=fc3fda29f660,39089,1731769023769 2024-11-16T15:00:14,790 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=64 updating hbase:meta row=7352dcdbf9811dcebc51a714a1de4f55, regionState=OPENING, regionLocation=fc3fda29f660,39089,1731769023769 2024-11-16T15:00:14,791 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=71 updating hbase:meta row=4f0d635e77a1e3dd04c4d107bb0f1564, regionState=OPENING, regionLocation=fc3fda29f660,39089,1731769023769 2024-11-16T15:00:14,792 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=70 updating hbase:meta row=b5566c7e10a0e001c7f58316bdd43c2f, regionState=OPENING, regionLocation=fc3fda29f660,42985,1731769023951 2024-11-16T15:00:14,792 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=426ff231e487ce9f8492776ca1dacc85, ASSIGN because future has completed 2024-11-16T15:00:14,793 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=73, ppid=66, state=RUNNABLE, hasLock=false; OpenRegionProcedure 426ff231e487ce9f8492776ca1dacc85, server=fc3fda29f660,39089,1731769023769}] 2024-11-16T15:00:14,794 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=69, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=33054df03c1be51dc860405880be6075, ASSIGN because future has completed 2024-11-16T15:00:14,795 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=72, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=c2acbc05ae7bb1607a5719b701b40383, ASSIGN because future has completed 2024-11-16T15:00:14,795 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=7352dcdbf9811dcebc51a714a1de4f55, ASSIGN because future has completed 2024-11-16T15:00:14,797 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=71, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=4f0d635e77a1e3dd04c4d107bb0f1564, ASSIGN because future has completed 2024-11-16T15:00:14,797 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=74, ppid=71, state=RUNNABLE, hasLock=false; OpenRegionProcedure 4f0d635e77a1e3dd04c4d107bb0f1564, server=fc3fda29f660,39089,1731769023769}] 2024-11-16T15:00:14,797 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=70, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=b5566c7e10a0e001c7f58316bdd43c2f, ASSIGN because future has completed 2024-11-16T15:00:14,800 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=75, ppid=70, state=RUNNABLE, hasLock=false; OpenRegionProcedure b5566c7e10a0e001c7f58316bdd43c2f, server=fc3fda29f660,42985,1731769023951}] 2024-11-16T15:00:14,802 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=76, ppid=64, state=RUNNABLE, hasLock=false; OpenRegionProcedure 7352dcdbf9811dcebc51a714a1de4f55, server=fc3fda29f660,39089,1731769023769}] 2024-11-16T15:00:14,804 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=77, ppid=72, state=RUNNABLE, hasLock=false; OpenRegionProcedure c2acbc05ae7bb1607a5719b701b40383, server=fc3fda29f660,39089,1731769023769}] 2024-11-16T15:00:14,806 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=78, ppid=69, state=RUNNABLE, hasLock=false; OpenRegionProcedure 33054df03c1be51dc860405880be6075, server=fc3fda29f660,39089,1731769023769}] 2024-11-16T15:00:14,811 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=65 updating hbase:meta row=7f5c9df3a544933227f58b052f743f72, regionState=OPENING, regionLocation=fc3fda29f660,42985,1731769023951 2024-11-16T15:00:14,811 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=68 updating hbase:meta row=7eb7b75d28b808c5c3f2f96e407b4347, regionState=OPENING, regionLocation=fc3fda29f660,42985,1731769023951 2024-11-16T15:00:14,812 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=67 updating hbase:meta row=fd3944fc2b78246a853fb2232c747c35, regionState=OPENING, regionLocation=fc3fda29f660,42985,1731769023951 2024-11-16T15:00:14,813 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=7f5c9df3a544933227f58b052f743f72, ASSIGN because future has completed 2024-11-16T15:00:14,817 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=79, ppid=65, state=RUNNABLE, hasLock=false; OpenRegionProcedure 7f5c9df3a544933227f58b052f743f72, server=fc3fda29f660,42985,1731769023951}] 2024-11-16T15:00:14,817 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=7eb7b75d28b808c5c3f2f96e407b4347, ASSIGN because future has completed 2024-11-16T15:00:14,818 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=fd3944fc2b78246a853fb2232c747c35, ASSIGN because future has completed 2024-11-16T15:00:14,821 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=80, ppid=67, state=RUNNABLE, hasLock=false; OpenRegionProcedure fd3944fc2b78246a853fb2232c747c35, server=fc3fda29f660,42985,1731769023951}] 2024-11-16T15:00:14,824 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=81, ppid=68, state=RUNNABLE, hasLock=false; OpenRegionProcedure 7eb7b75d28b808c5c3f2f96e407b4347, server=fc3fda29f660,42985,1731769023951}] 2024-11-16T15:00:14,950 INFO [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] handler.AssignRegionHandler(132): Open testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,yyy,1731769026392.426ff231e487ce9f8492776ca1dacc85. 2024-11-16T15:00:14,950 DEBUG [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] regionserver.HRegion(7752): Opening region: {ENCODED => 426ff231e487ce9f8492776ca1dacc85, NAME => 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,yyy,1731769026392.426ff231e487ce9f8492776ca1dacc85.', STARTKEY => 'yyy', ENDKEY => 'zzz'} 2024-11-16T15:00:14,951 DEBUG [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true 426ff231e487ce9f8492776ca1dacc85 2024-11-16T15:00:14,951 DEBUG [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] regionserver.HRegion(898): Instantiated testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,yyy,1731769026392.426ff231e487ce9f8492776ca1dacc85.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-16T15:00:14,951 DEBUG [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] regionserver.HRegion(7794): checking encryption for 426ff231e487ce9f8492776ca1dacc85 2024-11-16T15:00:14,951 DEBUG [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] regionserver.HRegion(7797): checking classloading for 426ff231e487ce9f8492776ca1dacc85 2024-11-16T15:00:14,953 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46925, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-16T15:00:14,953 INFO [StoreOpener-426ff231e487ce9f8492776ca1dacc85-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily11 of region 426ff231e487ce9f8492776ca1dacc85 2024-11-16T15:00:14,954 INFO [StoreOpener-426ff231e487ce9f8492776ca1dacc85-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 426ff231e487ce9f8492776ca1dacc85 columnFamilyName colfamily11 2024-11-16T15:00:14,954 DEBUG [StoreOpener-426ff231e487ce9f8492776ca1dacc85-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T15:00:14,955 INFO [StoreOpener-426ff231e487ce9f8492776ca1dacc85-1 {}] regionserver.HStore(327): Store=426ff231e487ce9f8492776ca1dacc85/colfamily11, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-16T15:00:14,955 INFO [StoreOpener-426ff231e487ce9f8492776ca1dacc85-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily21 of region 426ff231e487ce9f8492776ca1dacc85 2024-11-16T15:00:14,956 INFO [StoreOpener-426ff231e487ce9f8492776ca1dacc85-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 426ff231e487ce9f8492776ca1dacc85 columnFamilyName colfamily21 2024-11-16T15:00:14,956 DEBUG [StoreOpener-426ff231e487ce9f8492776ca1dacc85-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T15:00:14,956 INFO [StoreOpener-426ff231e487ce9f8492776ca1dacc85-1 {}] regionserver.HStore(327): Store=426ff231e487ce9f8492776ca1dacc85/colfamily21, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-16T15:00:14,957 INFO [StoreOpener-426ff231e487ce9f8492776ca1dacc85-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily31 of region 426ff231e487ce9f8492776ca1dacc85 2024-11-16T15:00:14,957 INFO [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] handler.AssignRegionHandler(132): Open testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,vvv,1731769026392.b5566c7e10a0e001c7f58316bdd43c2f. 2024-11-16T15:00:14,957 DEBUG [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(7752): Opening region: {ENCODED => b5566c7e10a0e001c7f58316bdd43c2f, NAME => 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,vvv,1731769026392.b5566c7e10a0e001c7f58316bdd43c2f.', STARTKEY => 'vvv', ENDKEY => 'www'} 2024-11-16T15:00:14,957 DEBUG [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true b5566c7e10a0e001c7f58316bdd43c2f 2024-11-16T15:00:14,957 DEBUG [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(898): Instantiated testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,vvv,1731769026392.b5566c7e10a0e001c7f58316bdd43c2f.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-16T15:00:14,957 INFO [StoreOpener-426ff231e487ce9f8492776ca1dacc85-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 426ff231e487ce9f8492776ca1dacc85 columnFamilyName colfamily31 2024-11-16T15:00:14,958 DEBUG [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(7794): checking encryption for b5566c7e10a0e001c7f58316bdd43c2f 2024-11-16T15:00:14,958 DEBUG [StoreOpener-426ff231e487ce9f8492776ca1dacc85-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T15:00:14,958 DEBUG [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(7797): checking classloading for b5566c7e10a0e001c7f58316bdd43c2f 2024-11-16T15:00:14,958 INFO [StoreOpener-426ff231e487ce9f8492776ca1dacc85-1 {}] regionserver.HStore(327): Store=426ff231e487ce9f8492776ca1dacc85/colfamily31, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-16T15:00:14,958 DEBUG [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] regionserver.HRegion(1038): replaying wal for 426ff231e487ce9f8492776ca1dacc85 2024-11-16T15:00:14,959 DEBUG [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42223/user/jenkins/test-data/a8176cb1-2ebb-b530-b3c7-53c9e53c186e/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/426ff231e487ce9f8492776ca1dacc85 2024-11-16T15:00:14,959 INFO [StoreOpener-b5566c7e10a0e001c7f58316bdd43c2f-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily11 of region b5566c7e10a0e001c7f58316bdd43c2f 2024-11-16T15:00:14,960 INFO [StoreOpener-b5566c7e10a0e001c7f58316bdd43c2f-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region b5566c7e10a0e001c7f58316bdd43c2f columnFamilyName colfamily11 2024-11-16T15:00:14,960 DEBUG [StoreOpener-b5566c7e10a0e001c7f58316bdd43c2f-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T15:00:14,960 INFO [StoreOpener-b5566c7e10a0e001c7f58316bdd43c2f-1 {}] regionserver.HStore(327): Store=b5566c7e10a0e001c7f58316bdd43c2f/colfamily11, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-16T15:00:14,961 INFO [StoreOpener-b5566c7e10a0e001c7f58316bdd43c2f-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily21 of region b5566c7e10a0e001c7f58316bdd43c2f 2024-11-16T15:00:14,961 DEBUG [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:42223/user/jenkins/test-data/a8176cb1-2ebb-b530-b3c7-53c9e53c186e/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/426ff231e487ce9f8492776ca1dacc85 2024-11-16T15:00:14,961 INFO [StoreOpener-b5566c7e10a0e001c7f58316bdd43c2f-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region b5566c7e10a0e001c7f58316bdd43c2f columnFamilyName colfamily21 2024-11-16T15:00:14,962 DEBUG [StoreOpener-b5566c7e10a0e001c7f58316bdd43c2f-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T15:00:14,962 INFO [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:42223/user/jenkins/test-data/a8176cb1-2ebb-b530-b3c7-53c9e53c186e/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/426ff231e487ce9f8492776ca1dacc85/recovered.edits/0000000000000000004 2024-11-16T15:00:14,962 INFO [StoreOpener-b5566c7e10a0e001c7f58316bdd43c2f-1 {}] regionserver.HStore(327): Store=b5566c7e10a0e001c7f58316bdd43c2f/colfamily21, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-16T15:00:14,963 INFO [StoreOpener-b5566c7e10a0e001c7f58316bdd43c2f-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily31 of region b5566c7e10a0e001c7f58316bdd43c2f 2024-11-16T15:00:14,963 INFO [StoreOpener-b5566c7e10a0e001c7f58316bdd43c2f-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region b5566c7e10a0e001c7f58316bdd43c2f columnFamilyName colfamily31 2024-11-16T15:00:14,964 DEBUG [StoreOpener-b5566c7e10a0e001c7f58316bdd43c2f-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T15:00:14,964 INFO [StoreOpener-b5566c7e10a0e001c7f58316bdd43c2f-1 {}] regionserver.HStore(327): Store=b5566c7e10a0e001c7f58316bdd43c2f/colfamily31, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-16T15:00:14,964 DEBUG [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(1038): replaying wal for b5566c7e10a0e001c7f58316bdd43c2f 2024-11-16T15:00:14,965 DEBUG [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42223/user/jenkins/test-data/a8176cb1-2ebb-b530-b3c7-53c9e53c186e/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/b5566c7e10a0e001c7f58316bdd43c2f 2024-11-16T15:00:14,967 DEBUG [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:42223/user/jenkins/test-data/a8176cb1-2ebb-b530-b3c7-53c9e53c186e/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/b5566c7e10a0e001c7f58316bdd43c2f 2024-11-16T15:00:14,968 DEBUG [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] regionserver.HRegion(5793): Applied 703, skipped 0, firstSequenceIdInLog=4, maxSequenceIdInLog=4, path=hdfs://localhost:42223/user/jenkins/test-data/a8176cb1-2ebb-b530-b3c7-53c9e53c186e/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/426ff231e487ce9f8492776ca1dacc85/recovered.edits/0000000000000000004 2024-11-16T15:00:14,968 INFO [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:42223/user/jenkins/test-data/a8176cb1-2ebb-b530-b3c7-53c9e53c186e/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/b5566c7e10a0e001c7f58316bdd43c2f/recovered.edits/0000000000000000004 2024-11-16T15:00:14,968 INFO [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] regionserver.HRegion(2902): Flushing 426ff231e487ce9f8492776ca1dacc85 3/3 column families, dataSize=32.95 KB heapSize=83.13 KB 2024-11-16T15:00:14,974 DEBUG [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(5793): Applied 703, skipped 0, firstSequenceIdInLog=4, maxSequenceIdInLog=4, path=hdfs://localhost:42223/user/jenkins/test-data/a8176cb1-2ebb-b530-b3c7-53c9e53c186e/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/b5566c7e10a0e001c7f58316bdd43c2f/recovered.edits/0000000000000000004 2024-11-16T15:00:14,974 INFO [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(2902): Flushing b5566c7e10a0e001c7f58316bdd43c2f 3/3 column families, dataSize=32.95 KB heapSize=83.13 KB 2024-11-16T15:00:14,996 DEBUG [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42223/user/jenkins/test-data/a8176cb1-2ebb-b530-b3c7-53c9e53c186e/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/426ff231e487ce9f8492776ca1dacc85/.tmp/colfamily11/6ba69d1a76f14c40b17d924ef1408c56 is 52, key is yyy/colfamily11:colfamily11/1731769030558/Put/seqid=0 2024-11-16T15:00:14,998 DEBUG [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42223/user/jenkins/test-data/a8176cb1-2ebb-b530-b3c7-53c9e53c186e/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/b5566c7e10a0e001c7f58316bdd43c2f/.tmp/colfamily11/bae32f7f27c04cebbddef474bc010e0c is 52, key is vvv/colfamily11:colfamily11/1731769030536/Put/seqid=0 2024-11-16T15:00:15,003 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40641 is added to blk_1073741878_1054 (size=39263) 2024-11-16T15:00:15,004 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36531 is added to blk_1073741878_1054 (size=39263) 2024-11-16T15:00:15,004 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41671 is added to blk_1073741878_1054 (size=39263) 2024-11-16T15:00:15,004 INFO [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=32.95 KB at sequenceid=4 (bloomFilter=false), to=hdfs://localhost:42223/user/jenkins/test-data/a8176cb1-2ebb-b530-b3c7-53c9e53c186e/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/426ff231e487ce9f8492776ca1dacc85/.tmp/colfamily11/6ba69d1a76f14c40b17d924ef1408c56 2024-11-16T15:00:15,008 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40641 is added to blk_1073741879_1055 (size=39263) 2024-11-16T15:00:15,011 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41671 is added to blk_1073741879_1055 (size=39263) 2024-11-16T15:00:15,011 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36531 is added to blk_1073741879_1055 (size=39263) 2024-11-16T15:00:15,011 INFO [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=32.95 KB at sequenceid=4 (bloomFilter=false), to=hdfs://localhost:42223/user/jenkins/test-data/a8176cb1-2ebb-b530-b3c7-53c9e53c186e/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/b5566c7e10a0e001c7f58316bdd43c2f/.tmp/colfamily11/bae32f7f27c04cebbddef474bc010e0c 2024-11-16T15:00:15,014 DEBUG [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42223/user/jenkins/test-data/a8176cb1-2ebb-b530-b3c7-53c9e53c186e/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/426ff231e487ce9f8492776ca1dacc85/.tmp/colfamily11/6ba69d1a76f14c40b17d924ef1408c56 as hdfs://localhost:42223/user/jenkins/test-data/a8176cb1-2ebb-b530-b3c7-53c9e53c186e/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/426ff231e487ce9f8492776ca1dacc85/colfamily11/6ba69d1a76f14c40b17d924ef1408c56 2024-11-16T15:00:15,019 DEBUG [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42223/user/jenkins/test-data/a8176cb1-2ebb-b530-b3c7-53c9e53c186e/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/b5566c7e10a0e001c7f58316bdd43c2f/.tmp/colfamily11/bae32f7f27c04cebbddef474bc010e0c as hdfs://localhost:42223/user/jenkins/test-data/a8176cb1-2ebb-b530-b3c7-53c9e53c186e/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/b5566c7e10a0e001c7f58316bdd43c2f/colfamily11/bae32f7f27c04cebbddef474bc010e0c 2024-11-16T15:00:15,021 INFO [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42223/user/jenkins/test-data/a8176cb1-2ebb-b530-b3c7-53c9e53c186e/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/426ff231e487ce9f8492776ca1dacc85/colfamily11/6ba69d1a76f14c40b17d924ef1408c56, entries=703, sequenceid=4, filesize=38.3 K 2024-11-16T15:00:15,021 INFO [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] regionserver.HRegion(3140): Finished flush of dataSize ~32.95 KB/33744, heapSize ~82.62 KB/84600, currentSize=0 B/0 for 426ff231e487ce9f8492776ca1dacc85 in 53ms, sequenceid=4, compaction requested=false; wal=null 2024-11-16T15:00:15,022 DEBUG [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:42223/user/jenkins/test-data/a8176cb1-2ebb-b530-b3c7-53c9e53c186e/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/426ff231e487ce9f8492776ca1dacc85/recovered.edits/0000000000000000004 2024-11-16T15:00:15,023 DEBUG [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] regionserver.HRegion(1048): stopping wal replay for 426ff231e487ce9f8492776ca1dacc85 2024-11-16T15:00:15,024 DEBUG [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] regionserver.HRegion(1060): Cleaning up temporary data for 426ff231e487ce9f8492776ca1dacc85 2024-11-16T15:00:15,024 DEBUG [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-16T15:00:15,026 INFO [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42223/user/jenkins/test-data/a8176cb1-2ebb-b530-b3c7-53c9e53c186e/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/b5566c7e10a0e001c7f58316bdd43c2f/colfamily11/bae32f7f27c04cebbddef474bc010e0c, entries=703, sequenceid=4, filesize=38.3 K 2024-11-16T15:00:15,026 DEBUG [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] regionserver.HRegion(1093): writing seq id for 426ff231e487ce9f8492776ca1dacc85 2024-11-16T15:00:15,027 INFO [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(3140): Finished flush of dataSize ~32.95 KB/33744, heapSize ~82.62 KB/84600, currentSize=0 B/0 for b5566c7e10a0e001c7f58316bdd43c2f in 53ms, sequenceid=4, compaction requested=false; wal=null 2024-11-16T15:00:15,027 DEBUG [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:42223/user/jenkins/test-data/a8176cb1-2ebb-b530-b3c7-53c9e53c186e/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/b5566c7e10a0e001c7f58316bdd43c2f/recovered.edits/0000000000000000004 2024-11-16T15:00:15,029 DEBUG [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(1048): stopping wal replay for b5566c7e10a0e001c7f58316bdd43c2f 2024-11-16T15:00:15,029 DEBUG [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(1060): Cleaning up temporary data for b5566c7e10a0e001c7f58316bdd43c2f 2024-11-16T15:00:15,029 DEBUG [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:42223/user/jenkins/test-data/a8176cb1-2ebb-b530-b3c7-53c9e53c186e/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/426ff231e487ce9f8492776ca1dacc85/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-16T15:00:15,030 INFO [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] regionserver.HRegion(1114): Opened 426ff231e487ce9f8492776ca1dacc85; next sequenceid=5; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=74715729, jitterRate=0.11335112154483795}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-16T15:00:15,030 DEBUG [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 426ff231e487ce9f8492776ca1dacc85 2024-11-16T15:00:15,030 DEBUG [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] regionserver.HRegion(1006): Region open journal for 426ff231e487ce9f8492776ca1dacc85: Running coprocessor pre-open hook at 1731769214951Writing region info on filesystem at 1731769214951Initializing all the Stores at 1731769214952 (+1 ms)Instantiating store for column family {NAME => 'colfamily11', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731769214952Instantiating store for column family {NAME => 'colfamily21', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731769214953 (+1 ms)Instantiating store for column family {NAME => 'colfamily31', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731769214953Obtaining lock to block concurrent updates at 1731769214968 (+15 ms)Preparing flush snapshotting stores in 426ff231e487ce9f8492776ca1dacc85 at 1731769214968Finished memstore snapshotting testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,yyy,1731769026392.426ff231e487ce9f8492776ca1dacc85., syncing WAL and waiting on mvcc, flushsize=dataSize=33744, getHeapSize=85080, getOffHeapSize=0, getCellsCount=703 at 1731769214968Flushing stores of testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,yyy,1731769026392.426ff231e487ce9f8492776ca1dacc85. at 1731769214968Flushing 426ff231e487ce9f8492776ca1dacc85/colfamily11: creating writer at 1731769214968Flushing 426ff231e487ce9f8492776ca1dacc85/colfamily11: appending metadata at 1731769214995 (+27 ms)Flushing 426ff231e487ce9f8492776ca1dacc85/colfamily11: closing flushed file at 1731769214996 (+1 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@7ccf26bf: reopening flushed file at 1731769215013 (+17 ms)Finished flush of dataSize ~32.95 KB/33744, heapSize ~82.62 KB/84600, currentSize=0 B/0 for 426ff231e487ce9f8492776ca1dacc85 in 53ms, sequenceid=4, compaction requested=false; wal=null at 1731769215021 (+8 ms)Cleaning up temporary data from old regions at 1731769215024 (+3 ms)Running coprocessor post-open hooks at 1731769215030 (+6 ms)Region opened successfully at 1731769215030 2024-11-16T15:00:15,030 DEBUG [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-16T15:00:15,031 INFO [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] regionserver.HRegionServer(2236): Post open deploy tasks for testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,yyy,1731769026392.426ff231e487ce9f8492776ca1dacc85., pid=73, masterSystemTime=1731769214946 2024-11-16T15:00:15,032 DEBUG [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(1093): writing seq id for b5566c7e10a0e001c7f58316bdd43c2f 2024-11-16T15:00:15,033 DEBUG [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] regionserver.HRegionServer(2266): Finished post open deploy task for testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,yyy,1731769026392.426ff231e487ce9f8492776ca1dacc85. 2024-11-16T15:00:15,034 INFO [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] handler.AssignRegionHandler(153): Opened testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,yyy,1731769026392.426ff231e487ce9f8492776ca1dacc85. 2024-11-16T15:00:15,034 INFO [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] handler.AssignRegionHandler(132): Open testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,,1731769026392.4f0d635e77a1e3dd04c4d107bb0f1564. 2024-11-16T15:00:15,034 DEBUG [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] regionserver.HRegion(7752): Opening region: {ENCODED => 4f0d635e77a1e3dd04c4d107bb0f1564, NAME => 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,,1731769026392.4f0d635e77a1e3dd04c4d107bb0f1564.', STARTKEY => '', ENDKEY => 'bbb'} 2024-11-16T15:00:15,034 DEBUG [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true 4f0d635e77a1e3dd04c4d107bb0f1564 2024-11-16T15:00:15,034 DEBUG [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] regionserver.HRegion(898): Instantiated testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,,1731769026392.4f0d635e77a1e3dd04c4d107bb0f1564.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-16T15:00:15,035 DEBUG [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] regionserver.HRegion(7794): checking encryption for 4f0d635e77a1e3dd04c4d107bb0f1564 2024-11-16T15:00:15,035 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=66 updating hbase:meta row=426ff231e487ce9f8492776ca1dacc85, regionState=OPEN, openSeqNum=5, regionLocation=fc3fda29f660,39089,1731769023769 2024-11-16T15:00:15,035 DEBUG [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] regionserver.HRegion(7797): checking classloading for 4f0d635e77a1e3dd04c4d107bb0f1564 2024-11-16T15:00:15,037 DEBUG [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:42223/user/jenkins/test-data/a8176cb1-2ebb-b530-b3c7-53c9e53c186e/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/b5566c7e10a0e001c7f58316bdd43c2f/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-16T15:00:15,037 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=73, ppid=66, state=RUNNABLE, hasLock=false; OpenRegionProcedure 426ff231e487ce9f8492776ca1dacc85, server=fc3fda29f660,39089,1731769023769 because future has completed 2024-11-16T15:00:15,037 INFO [StoreOpener-4f0d635e77a1e3dd04c4d107bb0f1564-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily11 of region 4f0d635e77a1e3dd04c4d107bb0f1564 2024-11-16T15:00:15,038 INFO [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(1114): Opened b5566c7e10a0e001c7f58316bdd43c2f; next sequenceid=5; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=68677412, jitterRate=0.023373186588287354}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-16T15:00:15,039 DEBUG [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(1122): Running coprocessor post-open hooks for b5566c7e10a0e001c7f58316bdd43c2f 2024-11-16T15:00:15,039 DEBUG [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(1006): Region open journal for b5566c7e10a0e001c7f58316bdd43c2f: Running coprocessor pre-open hook at 1731769214958Writing region info on filesystem at 1731769214958Initializing all the Stores at 1731769214959 (+1 ms)Instantiating store for column family {NAME => 'colfamily11', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731769214959Instantiating store for column family {NAME => 'colfamily21', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731769214959Instantiating store for column family {NAME => 'colfamily31', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731769214959Obtaining lock to block concurrent updates at 1731769214974 (+15 ms)Preparing flush snapshotting stores in b5566c7e10a0e001c7f58316bdd43c2f at 1731769214974Finished memstore snapshotting testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,vvv,1731769026392.b5566c7e10a0e001c7f58316bdd43c2f., syncing WAL and waiting on mvcc, flushsize=dataSize=33744, getHeapSize=85080, getOffHeapSize=0, getCellsCount=703 at 1731769214974Flushing stores of testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,vvv,1731769026392.b5566c7e10a0e001c7f58316bdd43c2f. at 1731769214974Flushing b5566c7e10a0e001c7f58316bdd43c2f/colfamily11: creating writer at 1731769214974Flushing b5566c7e10a0e001c7f58316bdd43c2f/colfamily11: appending metadata at 1731769214998 (+24 ms)Flushing b5566c7e10a0e001c7f58316bdd43c2f/colfamily11: closing flushed file at 1731769214998Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@78671e15: reopening flushed file at 1731769215018 (+20 ms)Finished flush of dataSize ~32.95 KB/33744, heapSize ~82.62 KB/84600, currentSize=0 B/0 for b5566c7e10a0e001c7f58316bdd43c2f in 53ms, sequenceid=4, compaction requested=false; wal=null at 1731769215027 (+9 ms)Cleaning up temporary data from old regions at 1731769215029 (+2 ms)Running coprocessor post-open hooks at 1731769215039 (+10 ms)Region opened successfully at 1731769215039 2024-11-16T15:00:15,039 INFO [StoreOpener-4f0d635e77a1e3dd04c4d107bb0f1564-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 4f0d635e77a1e3dd04c4d107bb0f1564 columnFamilyName colfamily11 2024-11-16T15:00:15,039 DEBUG [StoreOpener-4f0d635e77a1e3dd04c4d107bb0f1564-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T15:00:15,040 INFO [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegionServer(2236): Post open deploy tasks for testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,vvv,1731769026392.b5566c7e10a0e001c7f58316bdd43c2f., pid=75, masterSystemTime=1731769214951 2024-11-16T15:00:15,040 INFO [StoreOpener-4f0d635e77a1e3dd04c4d107bb0f1564-1 {}] regionserver.HStore(327): Store=4f0d635e77a1e3dd04c4d107bb0f1564/colfamily11, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-16T15:00:15,040 INFO [StoreOpener-4f0d635e77a1e3dd04c4d107bb0f1564-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily21 of region 4f0d635e77a1e3dd04c4d107bb0f1564 2024-11-16T15:00:15,041 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=73, resume processing ppid=66 2024-11-16T15:00:15,041 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=73, ppid=66, state=SUCCESS, hasLock=false; OpenRegionProcedure 426ff231e487ce9f8492776ca1dacc85, server=fc3fda29f660,39089,1731769023769 in 245 msec 2024-11-16T15:00:15,042 DEBUG [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegionServer(2266): Finished post open deploy task for testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,vvv,1731769026392.b5566c7e10a0e001c7f58316bdd43c2f. 2024-11-16T15:00:15,042 INFO [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] handler.AssignRegionHandler(153): Opened testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,vvv,1731769026392.b5566c7e10a0e001c7f58316bdd43c2f. 2024-11-16T15:00:15,042 INFO [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] handler.AssignRegionHandler(132): Open testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ppp,1731769026392.7f5c9df3a544933227f58b052f743f72. 2024-11-16T15:00:15,042 DEBUG [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(7752): Opening region: {ENCODED => 7f5c9df3a544933227f58b052f743f72, NAME => 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ppp,1731769026392.7f5c9df3a544933227f58b052f743f72.', STARTKEY => 'ppp', ENDKEY => 'qqq'} 2024-11-16T15:00:15,043 DEBUG [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true 7f5c9df3a544933227f58b052f743f72 2024-11-16T15:00:15,043 DEBUG [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(898): Instantiated testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ppp,1731769026392.7f5c9df3a544933227f58b052f743f72.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-16T15:00:15,043 DEBUG [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(7794): checking encryption for 7f5c9df3a544933227f58b052f743f72 2024-11-16T15:00:15,043 DEBUG [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(7797): checking classloading for 7f5c9df3a544933227f58b052f743f72 2024-11-16T15:00:15,043 INFO [StoreOpener-4f0d635e77a1e3dd04c4d107bb0f1564-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 4f0d635e77a1e3dd04c4d107bb0f1564 columnFamilyName colfamily21 2024-11-16T15:00:15,043 DEBUG [StoreOpener-4f0d635e77a1e3dd04c4d107bb0f1564-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T15:00:15,043 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=66, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=426ff231e487ce9f8492776ca1dacc85, ASSIGN in 412 msec 2024-11-16T15:00:15,044 INFO [StoreOpener-4f0d635e77a1e3dd04c4d107bb0f1564-1 {}] regionserver.HStore(327): Store=4f0d635e77a1e3dd04c4d107bb0f1564/colfamily21, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-16T15:00:15,044 INFO [StoreOpener-4f0d635e77a1e3dd04c4d107bb0f1564-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily31 of region 4f0d635e77a1e3dd04c4d107bb0f1564 2024-11-16T15:00:15,044 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=70 updating hbase:meta row=b5566c7e10a0e001c7f58316bdd43c2f, regionState=OPEN, openSeqNum=5, regionLocation=fc3fda29f660,42985,1731769023951 2024-11-16T15:00:15,044 INFO [StoreOpener-7f5c9df3a544933227f58b052f743f72-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily11 of region 7f5c9df3a544933227f58b052f743f72 2024-11-16T15:00:15,045 INFO [StoreOpener-4f0d635e77a1e3dd04c4d107bb0f1564-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 4f0d635e77a1e3dd04c4d107bb0f1564 columnFamilyName colfamily31 2024-11-16T15:00:15,045 DEBUG [StoreOpener-4f0d635e77a1e3dd04c4d107bb0f1564-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T15:00:15,045 INFO [StoreOpener-7f5c9df3a544933227f58b052f743f72-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 7f5c9df3a544933227f58b052f743f72 columnFamilyName colfamily11 2024-11-16T15:00:15,045 INFO [StoreOpener-4f0d635e77a1e3dd04c4d107bb0f1564-1 {}] regionserver.HStore(327): Store=4f0d635e77a1e3dd04c4d107bb0f1564/colfamily31, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-16T15:00:15,046 DEBUG [StoreOpener-7f5c9df3a544933227f58b052f743f72-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T15:00:15,046 DEBUG [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] regionserver.HRegion(1038): replaying wal for 4f0d635e77a1e3dd04c4d107bb0f1564 2024-11-16T15:00:15,046 INFO [StoreOpener-7f5c9df3a544933227f58b052f743f72-1 {}] regionserver.HStore(327): Store=7f5c9df3a544933227f58b052f743f72/colfamily11, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-16T15:00:15,047 INFO [StoreOpener-7f5c9df3a544933227f58b052f743f72-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily21 of region 7f5c9df3a544933227f58b052f743f72 2024-11-16T15:00:15,047 DEBUG [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42223/user/jenkins/test-data/a8176cb1-2ebb-b530-b3c7-53c9e53c186e/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/4f0d635e77a1e3dd04c4d107bb0f1564 2024-11-16T15:00:15,047 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=75, ppid=70, state=RUNNABLE, hasLock=false; OpenRegionProcedure b5566c7e10a0e001c7f58316bdd43c2f, server=fc3fda29f660,42985,1731769023951 because future has completed 2024-11-16T15:00:15,048 INFO [StoreOpener-7f5c9df3a544933227f58b052f743f72-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 7f5c9df3a544933227f58b052f743f72 columnFamilyName colfamily21 2024-11-16T15:00:15,048 DEBUG [StoreOpener-7f5c9df3a544933227f58b052f743f72-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T15:00:15,049 INFO [StoreOpener-7f5c9df3a544933227f58b052f743f72-1 {}] regionserver.HStore(327): Store=7f5c9df3a544933227f58b052f743f72/colfamily21, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-16T15:00:15,049 DEBUG [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:42223/user/jenkins/test-data/a8176cb1-2ebb-b530-b3c7-53c9e53c186e/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/4f0d635e77a1e3dd04c4d107bb0f1564 2024-11-16T15:00:15,049 INFO [StoreOpener-7f5c9df3a544933227f58b052f743f72-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily31 of region 7f5c9df3a544933227f58b052f743f72 2024-11-16T15:00:15,050 INFO [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:42223/user/jenkins/test-data/a8176cb1-2ebb-b530-b3c7-53c9e53c186e/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/4f0d635e77a1e3dd04c4d107bb0f1564/recovered.edits/0000000000000000004 2024-11-16T15:00:15,051 INFO [StoreOpener-7f5c9df3a544933227f58b052f743f72-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 7f5c9df3a544933227f58b052f743f72 columnFamilyName colfamily31 2024-11-16T15:00:15,051 DEBUG [StoreOpener-7f5c9df3a544933227f58b052f743f72-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T15:00:15,051 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=75, resume processing ppid=70 2024-11-16T15:00:15,051 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=75, ppid=70, state=SUCCESS, hasLock=false; OpenRegionProcedure b5566c7e10a0e001c7f58316bdd43c2f, server=fc3fda29f660,42985,1731769023951 in 249 msec 2024-11-16T15:00:15,052 INFO [StoreOpener-7f5c9df3a544933227f58b052f743f72-1 {}] regionserver.HStore(327): Store=7f5c9df3a544933227f58b052f743f72/colfamily31, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-16T15:00:15,052 DEBUG [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(1038): replaying wal for 7f5c9df3a544933227f58b052f743f72 2024-11-16T15:00:15,054 DEBUG [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42223/user/jenkins/test-data/a8176cb1-2ebb-b530-b3c7-53c9e53c186e/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/7f5c9df3a544933227f58b052f743f72 2024-11-16T15:00:15,055 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=70, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=b5566c7e10a0e001c7f58316bdd43c2f, ASSIGN in 423 msec 2024-11-16T15:00:15,057 DEBUG [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] regionserver.HRegion(5793): Applied 703, skipped 0, firstSequenceIdInLog=4, maxSequenceIdInLog=4, path=hdfs://localhost:42223/user/jenkins/test-data/a8176cb1-2ebb-b530-b3c7-53c9e53c186e/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/4f0d635e77a1e3dd04c4d107bb0f1564/recovered.edits/0000000000000000004 2024-11-16T15:00:15,057 INFO [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] regionserver.HRegion(2902): Flushing 4f0d635e77a1e3dd04c4d107bb0f1564 3/3 column families, dataSize=32.95 KB heapSize=83.13 KB 2024-11-16T15:00:15,061 DEBUG [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:42223/user/jenkins/test-data/a8176cb1-2ebb-b530-b3c7-53c9e53c186e/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/7f5c9df3a544933227f58b052f743f72 2024-11-16T15:00:15,061 INFO [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:42223/user/jenkins/test-data/a8176cb1-2ebb-b530-b3c7-53c9e53c186e/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/7f5c9df3a544933227f58b052f743f72/recovered.edits/0000000000000000004 2024-11-16T15:00:15,068 DEBUG [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(5793): Applied 703, skipped 0, firstSequenceIdInLog=4, maxSequenceIdInLog=4, path=hdfs://localhost:42223/user/jenkins/test-data/a8176cb1-2ebb-b530-b3c7-53c9e53c186e/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/7f5c9df3a544933227f58b052f743f72/recovered.edits/0000000000000000004 2024-11-16T15:00:15,068 INFO [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(2902): Flushing 7f5c9df3a544933227f58b052f743f72 3/3 column families, dataSize=32.95 KB heapSize=83.13 KB 2024-11-16T15:00:15,081 DEBUG [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42223/user/jenkins/test-data/a8176cb1-2ebb-b530-b3c7-53c9e53c186e/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/4f0d635e77a1e3dd04c4d107bb0f1564/.tmp/colfamily11/b1035690586d45fbad4b8c40c38aef13 is 52, key is aaa/colfamily11:colfamily11/1731769030292/Put/seqid=0 2024-11-16T15:00:15,086 DEBUG [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42223/user/jenkins/test-data/a8176cb1-2ebb-b530-b3c7-53c9e53c186e/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/7f5c9df3a544933227f58b052f743f72/.tmp/colfamily11/6ec2bed3066544d2a03273d27ca99aff is 52, key is ppp/colfamily11:colfamily11/1731769030467/Put/seqid=0 2024-11-16T15:00:15,091 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40641 is added to blk_1073741880_1056 (size=39263) 2024-11-16T15:00:15,091 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41671 is added to blk_1073741880_1056 (size=39263) 2024-11-16T15:00:15,092 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36531 is added to blk_1073741880_1056 (size=39263) 2024-11-16T15:00:15,092 INFO [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=32.95 KB at sequenceid=4 (bloomFilter=false), to=hdfs://localhost:42223/user/jenkins/test-data/a8176cb1-2ebb-b530-b3c7-53c9e53c186e/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/4f0d635e77a1e3dd04c4d107bb0f1564/.tmp/colfamily11/b1035690586d45fbad4b8c40c38aef13 2024-11-16T15:00:15,097 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41671 is added to blk_1073741881_1057 (size=39263) 2024-11-16T15:00:15,097 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36531 is added to blk_1073741881_1057 (size=39263) 2024-11-16T15:00:15,098 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40641 is added to blk_1073741881_1057 (size=39263) 2024-11-16T15:00:15,099 INFO [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=32.95 KB at sequenceid=4 (bloomFilter=false), to=hdfs://localhost:42223/user/jenkins/test-data/a8176cb1-2ebb-b530-b3c7-53c9e53c186e/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/7f5c9df3a544933227f58b052f743f72/.tmp/colfamily11/6ec2bed3066544d2a03273d27ca99aff 2024-11-16T15:00:15,105 DEBUG [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42223/user/jenkins/test-data/a8176cb1-2ebb-b530-b3c7-53c9e53c186e/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/4f0d635e77a1e3dd04c4d107bb0f1564/.tmp/colfamily11/b1035690586d45fbad4b8c40c38aef13 as hdfs://localhost:42223/user/jenkins/test-data/a8176cb1-2ebb-b530-b3c7-53c9e53c186e/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/4f0d635e77a1e3dd04c4d107bb0f1564/colfamily11/b1035690586d45fbad4b8c40c38aef13 2024-11-16T15:00:15,108 DEBUG [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42223/user/jenkins/test-data/a8176cb1-2ebb-b530-b3c7-53c9e53c186e/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/7f5c9df3a544933227f58b052f743f72/.tmp/colfamily11/6ec2bed3066544d2a03273d27ca99aff as hdfs://localhost:42223/user/jenkins/test-data/a8176cb1-2ebb-b530-b3c7-53c9e53c186e/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/7f5c9df3a544933227f58b052f743f72/colfamily11/6ec2bed3066544d2a03273d27ca99aff 2024-11-16T15:00:15,113 INFO [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42223/user/jenkins/test-data/a8176cb1-2ebb-b530-b3c7-53c9e53c186e/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/4f0d635e77a1e3dd04c4d107bb0f1564/colfamily11/b1035690586d45fbad4b8c40c38aef13, entries=703, sequenceid=4, filesize=38.3 K 2024-11-16T15:00:15,114 INFO [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] regionserver.HRegion(3140): Finished flush of dataSize ~32.95 KB/33744, heapSize ~82.62 KB/84600, currentSize=0 B/0 for 4f0d635e77a1e3dd04c4d107bb0f1564 in 57ms, sequenceid=4, compaction requested=false; wal=null 2024-11-16T15:00:15,115 DEBUG [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:42223/user/jenkins/test-data/a8176cb1-2ebb-b530-b3c7-53c9e53c186e/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/4f0d635e77a1e3dd04c4d107bb0f1564/recovered.edits/0000000000000000004 2024-11-16T15:00:15,116 INFO [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42223/user/jenkins/test-data/a8176cb1-2ebb-b530-b3c7-53c9e53c186e/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/7f5c9df3a544933227f58b052f743f72/colfamily11/6ec2bed3066544d2a03273d27ca99aff, entries=703, sequenceid=4, filesize=38.3 K 2024-11-16T15:00:15,116 INFO [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(3140): Finished flush of dataSize ~32.95 KB/33744, heapSize ~82.62 KB/84600, currentSize=0 B/0 for 7f5c9df3a544933227f58b052f743f72 in 48ms, sequenceid=4, compaction requested=false; wal=null 2024-11-16T15:00:15,116 DEBUG [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] regionserver.HRegion(1048): stopping wal replay for 4f0d635e77a1e3dd04c4d107bb0f1564 2024-11-16T15:00:15,116 DEBUG [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] regionserver.HRegion(1060): Cleaning up temporary data for 4f0d635e77a1e3dd04c4d107bb0f1564 2024-11-16T15:00:15,117 DEBUG [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:42223/user/jenkins/test-data/a8176cb1-2ebb-b530-b3c7-53c9e53c186e/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/7f5c9df3a544933227f58b052f743f72/recovered.edits/0000000000000000004 2024-11-16T15:00:15,118 DEBUG [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-16T15:00:15,118 DEBUG [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(1048): stopping wal replay for 7f5c9df3a544933227f58b052f743f72 2024-11-16T15:00:15,118 DEBUG [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(1060): Cleaning up temporary data for 7f5c9df3a544933227f58b052f743f72 2024-11-16T15:00:15,119 DEBUG [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-16T15:00:15,120 DEBUG [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] regionserver.HRegion(1093): writing seq id for 4f0d635e77a1e3dd04c4d107bb0f1564 2024-11-16T15:00:15,121 DEBUG [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(1093): writing seq id for 7f5c9df3a544933227f58b052f743f72 2024-11-16T15:00:15,123 DEBUG [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:42223/user/jenkins/test-data/a8176cb1-2ebb-b530-b3c7-53c9e53c186e/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/4f0d635e77a1e3dd04c4d107bb0f1564/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-16T15:00:15,123 DEBUG [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:42223/user/jenkins/test-data/a8176cb1-2ebb-b530-b3c7-53c9e53c186e/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/7f5c9df3a544933227f58b052f743f72/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-16T15:00:15,124 INFO [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] regionserver.HRegion(1114): Opened 4f0d635e77a1e3dd04c4d107bb0f1564; next sequenceid=5; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=63565683, jitterRate=-0.05279751121997833}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-16T15:00:15,124 INFO [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(1114): Opened 7f5c9df3a544933227f58b052f743f72; next sequenceid=5; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=72431947, jitterRate=0.07932011783123016}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-16T15:00:15,124 DEBUG [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 4f0d635e77a1e3dd04c4d107bb0f1564 2024-11-16T15:00:15,124 DEBUG [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 7f5c9df3a544933227f58b052f743f72 2024-11-16T15:00:15,125 DEBUG [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(1006): Region open journal for 7f5c9df3a544933227f58b052f743f72: Running coprocessor pre-open hook at 1731769215043Writing region info on filesystem at 1731769215043Initializing all the Stores at 1731769215044 (+1 ms)Instantiating store for column family {NAME => 'colfamily11', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731769215044Instantiating store for column family {NAME => 'colfamily21', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731769215044Instantiating store for column family {NAME => 'colfamily31', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731769215044Obtaining lock to block concurrent updates at 1731769215068 (+24 ms)Preparing flush snapshotting stores in 7f5c9df3a544933227f58b052f743f72 at 1731769215068Finished memstore snapshotting testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ppp,1731769026392.7f5c9df3a544933227f58b052f743f72., syncing WAL and waiting on mvcc, flushsize=dataSize=33744, getHeapSize=85080, getOffHeapSize=0, getCellsCount=703 at 1731769215069 (+1 ms)Flushing stores of testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ppp,1731769026392.7f5c9df3a544933227f58b052f743f72. at 1731769215069Flushing 7f5c9df3a544933227f58b052f743f72/colfamily11: creating writer at 1731769215069Flushing 7f5c9df3a544933227f58b052f743f72/colfamily11: appending metadata at 1731769215085 (+16 ms)Flushing 7f5c9df3a544933227f58b052f743f72/colfamily11: closing flushed file at 1731769215085Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@572c9c28: reopening flushed file at 1731769215107 (+22 ms)Finished flush of dataSize ~32.95 KB/33744, heapSize ~82.62 KB/84600, currentSize=0 B/0 for 7f5c9df3a544933227f58b052f743f72 in 48ms, sequenceid=4, compaction requested=false; wal=null at 1731769215116 (+9 ms)Cleaning up temporary data from old regions at 1731769215118 (+2 ms)Running coprocessor post-open hooks at 1731769215124 (+6 ms)Region opened successfully at 1731769215124 2024-11-16T15:00:15,125 DEBUG [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] regionserver.HRegion(1006): Region open journal for 4f0d635e77a1e3dd04c4d107bb0f1564: Running coprocessor pre-open hook at 1731769215035Writing region info on filesystem at 1731769215035Initializing all the Stores at 1731769215037 (+2 ms)Instantiating store for column family {NAME => 'colfamily11', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731769215037Instantiating store for column family {NAME => 'colfamily21', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731769215037Instantiating store for column family {NAME => 'colfamily31', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731769215037Obtaining lock to block concurrent updates at 1731769215057 (+20 ms)Preparing flush snapshotting stores in 4f0d635e77a1e3dd04c4d107bb0f1564 at 1731769215057Finished memstore snapshotting testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,,1731769026392.4f0d635e77a1e3dd04c4d107bb0f1564., syncing WAL and waiting on mvcc, flushsize=dataSize=33744, getHeapSize=85080, getOffHeapSize=0, getCellsCount=703 at 1731769215058 (+1 ms)Flushing stores of testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,,1731769026392.4f0d635e77a1e3dd04c4d107bb0f1564. at 1731769215058Flushing 4f0d635e77a1e3dd04c4d107bb0f1564/colfamily11: creating writer at 1731769215058Flushing 4f0d635e77a1e3dd04c4d107bb0f1564/colfamily11: appending metadata at 1731769215081 (+23 ms)Flushing 4f0d635e77a1e3dd04c4d107bb0f1564/colfamily11: closing flushed file at 1731769215081Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@450457b8: reopening flushed file at 1731769215101 (+20 ms)Finished flush of dataSize ~32.95 KB/33744, heapSize ~82.62 KB/84600, currentSize=0 B/0 for 4f0d635e77a1e3dd04c4d107bb0f1564 in 57ms, sequenceid=4, compaction requested=false; wal=null at 1731769215114 (+13 ms)Cleaning up temporary data from old regions at 1731769215116 (+2 ms)Running coprocessor post-open hooks at 1731769215124 (+8 ms)Region opened successfully at 1731769215124 2024-11-16T15:00:15,126 INFO [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] regionserver.HRegionServer(2236): Post open deploy tasks for testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,,1731769026392.4f0d635e77a1e3dd04c4d107bb0f1564., pid=74, masterSystemTime=1731769214946 2024-11-16T15:00:15,126 INFO [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegionServer(2236): Post open deploy tasks for testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ppp,1731769026392.7f5c9df3a544933227f58b052f743f72., pid=79, masterSystemTime=1731769214951 2024-11-16T15:00:15,127 DEBUG [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegionServer(2266): Finished post open deploy task for testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ppp,1731769026392.7f5c9df3a544933227f58b052f743f72. 2024-11-16T15:00:15,128 INFO [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] handler.AssignRegionHandler(153): Opened testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ppp,1731769026392.7f5c9df3a544933227f58b052f743f72. 2024-11-16T15:00:15,128 INFO [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] handler.AssignRegionHandler(132): Open testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,eee,1731769026392.fd3944fc2b78246a853fb2232c747c35. 2024-11-16T15:00:15,128 DEBUG [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.HRegion(7752): Opening region: {ENCODED => fd3944fc2b78246a853fb2232c747c35, NAME => 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,eee,1731769026392.fd3944fc2b78246a853fb2232c747c35.', STARTKEY => 'eee', ENDKEY => 'fff'} 2024-11-16T15:00:15,128 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=65 updating hbase:meta row=7f5c9df3a544933227f58b052f743f72, regionState=OPEN, openSeqNum=5, regionLocation=fc3fda29f660,42985,1731769023951 2024-11-16T15:00:15,128 DEBUG [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true fd3944fc2b78246a853fb2232c747c35 2024-11-16T15:00:15,128 DEBUG [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] regionserver.HRegionServer(2266): Finished post open deploy task for testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,,1731769026392.4f0d635e77a1e3dd04c4d107bb0f1564. 2024-11-16T15:00:15,128 DEBUG [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.HRegion(898): Instantiated testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,eee,1731769026392.fd3944fc2b78246a853fb2232c747c35.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-16T15:00:15,128 INFO [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] handler.AssignRegionHandler(153): Opened testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,,1731769026392.4f0d635e77a1e3dd04c4d107bb0f1564. 2024-11-16T15:00:15,129 DEBUG [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.HRegion(7794): checking encryption for fd3944fc2b78246a853fb2232c747c35 2024-11-16T15:00:15,129 INFO [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] handler.AssignRegionHandler(132): Open testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,iii,1731769026392.7352dcdbf9811dcebc51a714a1de4f55. 2024-11-16T15:00:15,129 DEBUG [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.HRegion(7797): checking classloading for fd3944fc2b78246a853fb2232c747c35 2024-11-16T15:00:15,129 DEBUG [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(7752): Opening region: {ENCODED => 7352dcdbf9811dcebc51a714a1de4f55, NAME => 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,iii,1731769026392.7352dcdbf9811dcebc51a714a1de4f55.', STARTKEY => 'iii', ENDKEY => 'jjj'} 2024-11-16T15:00:15,129 DEBUG [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true 7352dcdbf9811dcebc51a714a1de4f55 2024-11-16T15:00:15,129 DEBUG [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(898): Instantiated testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,iii,1731769026392.7352dcdbf9811dcebc51a714a1de4f55.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-16T15:00:15,129 DEBUG [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(7794): checking encryption for 7352dcdbf9811dcebc51a714a1de4f55 2024-11-16T15:00:15,129 DEBUG [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(7797): checking classloading for 7352dcdbf9811dcebc51a714a1de4f55 2024-11-16T15:00:15,131 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=79, ppid=65, state=RUNNABLE, hasLock=false; OpenRegionProcedure 7f5c9df3a544933227f58b052f743f72, server=fc3fda29f660,42985,1731769023951 because future has completed 2024-11-16T15:00:15,131 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=71 updating hbase:meta row=4f0d635e77a1e3dd04c4d107bb0f1564, regionState=OPEN, openSeqNum=5, regionLocation=fc3fda29f660,39089,1731769023769 2024-11-16T15:00:15,131 INFO [StoreOpener-fd3944fc2b78246a853fb2232c747c35-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily11 of region fd3944fc2b78246a853fb2232c747c35 2024-11-16T15:00:15,131 INFO [StoreOpener-7352dcdbf9811dcebc51a714a1de4f55-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily11 of region 7352dcdbf9811dcebc51a714a1de4f55 2024-11-16T15:00:15,132 INFO [StoreOpener-7352dcdbf9811dcebc51a714a1de4f55-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 7352dcdbf9811dcebc51a714a1de4f55 columnFamilyName colfamily11 2024-11-16T15:00:15,133 DEBUG [StoreOpener-7352dcdbf9811dcebc51a714a1de4f55-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T15:00:15,133 INFO [StoreOpener-fd3944fc2b78246a853fb2232c747c35-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region fd3944fc2b78246a853fb2232c747c35 columnFamilyName colfamily11 2024-11-16T15:00:15,134 DEBUG [StoreOpener-fd3944fc2b78246a853fb2232c747c35-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T15:00:15,134 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=74, ppid=71, state=RUNNABLE, hasLock=false; OpenRegionProcedure 4f0d635e77a1e3dd04c4d107bb0f1564, server=fc3fda29f660,39089,1731769023769 because future has completed 2024-11-16T15:00:15,134 INFO [StoreOpener-7352dcdbf9811dcebc51a714a1de4f55-1 {}] regionserver.HStore(327): Store=7352dcdbf9811dcebc51a714a1de4f55/colfamily11, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-16T15:00:15,134 INFO [StoreOpener-7352dcdbf9811dcebc51a714a1de4f55-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily21 of region 7352dcdbf9811dcebc51a714a1de4f55 2024-11-16T15:00:15,134 INFO [StoreOpener-fd3944fc2b78246a853fb2232c747c35-1 {}] regionserver.HStore(327): Store=fd3944fc2b78246a853fb2232c747c35/colfamily11, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-16T15:00:15,134 INFO [StoreOpener-fd3944fc2b78246a853fb2232c747c35-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily21 of region fd3944fc2b78246a853fb2232c747c35 2024-11-16T15:00:15,135 INFO [StoreOpener-7352dcdbf9811dcebc51a714a1de4f55-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 7352dcdbf9811dcebc51a714a1de4f55 columnFamilyName colfamily21 2024-11-16T15:00:15,135 DEBUG [StoreOpener-7352dcdbf9811dcebc51a714a1de4f55-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T15:00:15,135 INFO [StoreOpener-fd3944fc2b78246a853fb2232c747c35-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region fd3944fc2b78246a853fb2232c747c35 columnFamilyName colfamily21 2024-11-16T15:00:15,136 DEBUG [StoreOpener-fd3944fc2b78246a853fb2232c747c35-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T15:00:15,136 INFO [StoreOpener-7352dcdbf9811dcebc51a714a1de4f55-1 {}] regionserver.HStore(327): Store=7352dcdbf9811dcebc51a714a1de4f55/colfamily21, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-16T15:00:15,136 INFO [StoreOpener-7352dcdbf9811dcebc51a714a1de4f55-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily31 of region 7352dcdbf9811dcebc51a714a1de4f55 2024-11-16T15:00:15,136 INFO [StoreOpener-fd3944fc2b78246a853fb2232c747c35-1 {}] regionserver.HStore(327): Store=fd3944fc2b78246a853fb2232c747c35/colfamily21, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-16T15:00:15,136 INFO [StoreOpener-fd3944fc2b78246a853fb2232c747c35-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily31 of region fd3944fc2b78246a853fb2232c747c35 2024-11-16T15:00:15,137 INFO [StoreOpener-7352dcdbf9811dcebc51a714a1de4f55-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 7352dcdbf9811dcebc51a714a1de4f55 columnFamilyName colfamily31 2024-11-16T15:00:15,137 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=79, resume processing ppid=65 2024-11-16T15:00:15,137 DEBUG [StoreOpener-7352dcdbf9811dcebc51a714a1de4f55-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T15:00:15,137 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=79, ppid=65, state=SUCCESS, hasLock=false; OpenRegionProcedure 7f5c9df3a544933227f58b052f743f72, server=fc3fda29f660,42985,1731769023951 in 317 msec 2024-11-16T15:00:15,137 INFO [StoreOpener-fd3944fc2b78246a853fb2232c747c35-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region fd3944fc2b78246a853fb2232c747c35 columnFamilyName colfamily31 2024-11-16T15:00:15,137 INFO [StoreOpener-7352dcdbf9811dcebc51a714a1de4f55-1 {}] regionserver.HStore(327): Store=7352dcdbf9811dcebc51a714a1de4f55/colfamily31, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-16T15:00:15,137 DEBUG [StoreOpener-fd3944fc2b78246a853fb2232c747c35-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T15:00:15,138 DEBUG [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(1038): replaying wal for 7352dcdbf9811dcebc51a714a1de4f55 2024-11-16T15:00:15,138 INFO [StoreOpener-fd3944fc2b78246a853fb2232c747c35-1 {}] regionserver.HStore(327): Store=fd3944fc2b78246a853fb2232c747c35/colfamily31, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-16T15:00:15,138 DEBUG [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.HRegion(1038): replaying wal for fd3944fc2b78246a853fb2232c747c35 2024-11-16T15:00:15,138 DEBUG [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42223/user/jenkins/test-data/a8176cb1-2ebb-b530-b3c7-53c9e53c186e/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/7352dcdbf9811dcebc51a714a1de4f55 2024-11-16T15:00:15,139 DEBUG [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42223/user/jenkins/test-data/a8176cb1-2ebb-b530-b3c7-53c9e53c186e/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/fd3944fc2b78246a853fb2232c747c35 2024-11-16T15:00:15,139 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=65, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=7f5c9df3a544933227f58b052f743f72, ASSIGN in 508 msec 2024-11-16T15:00:15,140 DEBUG [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:42223/user/jenkins/test-data/a8176cb1-2ebb-b530-b3c7-53c9e53c186e/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/7352dcdbf9811dcebc51a714a1de4f55 2024-11-16T15:00:15,140 DEBUG [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:42223/user/jenkins/test-data/a8176cb1-2ebb-b530-b3c7-53c9e53c186e/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/fd3944fc2b78246a853fb2232c747c35 2024-11-16T15:00:15,141 INFO [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:42223/user/jenkins/test-data/a8176cb1-2ebb-b530-b3c7-53c9e53c186e/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/7352dcdbf9811dcebc51a714a1de4f55/recovered.edits/0000000000000000004 2024-11-16T15:00:15,141 INFO [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:42223/user/jenkins/test-data/a8176cb1-2ebb-b530-b3c7-53c9e53c186e/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/fd3944fc2b78246a853fb2232c747c35/recovered.edits/0000000000000000004 2024-11-16T15:00:15,143 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=74, resume processing ppid=71 2024-11-16T15:00:15,143 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=74, ppid=71, state=SUCCESS, hasLock=false; OpenRegionProcedure 4f0d635e77a1e3dd04c4d107bb0f1564, server=fc3fda29f660,39089,1731769023769 in 343 msec 2024-11-16T15:00:15,145 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=71, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=4f0d635e77a1e3dd04c4d107bb0f1564, ASSIGN in 514 msec 2024-11-16T15:00:15,146 DEBUG [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(5793): Applied 703, skipped 0, firstSequenceIdInLog=4, maxSequenceIdInLog=4, path=hdfs://localhost:42223/user/jenkins/test-data/a8176cb1-2ebb-b530-b3c7-53c9e53c186e/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/7352dcdbf9811dcebc51a714a1de4f55/recovered.edits/0000000000000000004 2024-11-16T15:00:15,147 INFO [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(2902): Flushing 7352dcdbf9811dcebc51a714a1de4f55 3/3 column families, dataSize=32.95 KB heapSize=83.13 KB 2024-11-16T15:00:15,147 DEBUG [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.HRegion(5793): Applied 703, skipped 0, firstSequenceIdInLog=4, maxSequenceIdInLog=4, path=hdfs://localhost:42223/user/jenkins/test-data/a8176cb1-2ebb-b530-b3c7-53c9e53c186e/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/fd3944fc2b78246a853fb2232c747c35/recovered.edits/0000000000000000004 2024-11-16T15:00:15,148 INFO [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.HRegion(2902): Flushing fd3944fc2b78246a853fb2232c747c35 3/3 column families, dataSize=32.95 KB heapSize=83.13 KB 2024-11-16T15:00:15,164 DEBUG [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42223/user/jenkins/test-data/a8176cb1-2ebb-b530-b3c7-53c9e53c186e/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/fd3944fc2b78246a853fb2232c747c35/.tmp/colfamily11/1fb29abe9e494d2b965ebb2ab1f0413c is 52, key is eee/colfamily11:colfamily11/1731769030333/Put/seqid=0 2024-11-16T15:00:15,165 DEBUG [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42223/user/jenkins/test-data/a8176cb1-2ebb-b530-b3c7-53c9e53c186e/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/7352dcdbf9811dcebc51a714a1de4f55/.tmp/colfamily11/7e109ae424e34ce19422529c9093592f is 52, key is iii/colfamily11:colfamily11/1731769030373/Put/seqid=0 2024-11-16T15:00:15,177 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41671 is added to blk_1073741883_1059 (size=39263) 2024-11-16T15:00:15,178 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36531 is added to blk_1073741882_1058 (size=39263) 2024-11-16T15:00:15,178 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41671 is added to blk_1073741882_1058 (size=39263) 2024-11-16T15:00:15,178 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40641 is added to blk_1073741882_1058 (size=39263) 2024-11-16T15:00:15,178 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36531 is added to blk_1073741883_1059 (size=39263) 2024-11-16T15:00:15,178 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40641 is added to blk_1073741883_1059 (size=39263) 2024-11-16T15:00:15,179 INFO [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=32.95 KB at sequenceid=4 (bloomFilter=false), to=hdfs://localhost:42223/user/jenkins/test-data/a8176cb1-2ebb-b530-b3c7-53c9e53c186e/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/7352dcdbf9811dcebc51a714a1de4f55/.tmp/colfamily11/7e109ae424e34ce19422529c9093592f 2024-11-16T15:00:15,186 DEBUG [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42223/user/jenkins/test-data/a8176cb1-2ebb-b530-b3c7-53c9e53c186e/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/7352dcdbf9811dcebc51a714a1de4f55/.tmp/colfamily11/7e109ae424e34ce19422529c9093592f as hdfs://localhost:42223/user/jenkins/test-data/a8176cb1-2ebb-b530-b3c7-53c9e53c186e/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/7352dcdbf9811dcebc51a714a1de4f55/colfamily11/7e109ae424e34ce19422529c9093592f 2024-11-16T15:00:15,194 INFO [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42223/user/jenkins/test-data/a8176cb1-2ebb-b530-b3c7-53c9e53c186e/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/7352dcdbf9811dcebc51a714a1de4f55/colfamily11/7e109ae424e34ce19422529c9093592f, entries=703, sequenceid=4, filesize=38.3 K 2024-11-16T15:00:15,194 INFO [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(3140): Finished flush of dataSize ~32.95 KB/33744, heapSize ~82.62 KB/84600, currentSize=0 B/0 for 7352dcdbf9811dcebc51a714a1de4f55 in 48ms, sequenceid=4, compaction requested=false; wal=null 2024-11-16T15:00:15,195 DEBUG [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:42223/user/jenkins/test-data/a8176cb1-2ebb-b530-b3c7-53c9e53c186e/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/7352dcdbf9811dcebc51a714a1de4f55/recovered.edits/0000000000000000004 2024-11-16T15:00:15,196 DEBUG [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(1048): stopping wal replay for 7352dcdbf9811dcebc51a714a1de4f55 2024-11-16T15:00:15,196 DEBUG [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(1060): Cleaning up temporary data for 7352dcdbf9811dcebc51a714a1de4f55 2024-11-16T15:00:15,197 DEBUG [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-16T15:00:15,198 DEBUG [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(1093): writing seq id for 7352dcdbf9811dcebc51a714a1de4f55 2024-11-16T15:00:15,201 DEBUG [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:42223/user/jenkins/test-data/a8176cb1-2ebb-b530-b3c7-53c9e53c186e/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/7352dcdbf9811dcebc51a714a1de4f55/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-16T15:00:15,202 INFO [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(1114): Opened 7352dcdbf9811dcebc51a714a1de4f55; next sequenceid=5; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=67658077, jitterRate=0.008183911442756653}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-16T15:00:15,202 DEBUG [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 7352dcdbf9811dcebc51a714a1de4f55 2024-11-16T15:00:15,202 DEBUG [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(1006): Region open journal for 7352dcdbf9811dcebc51a714a1de4f55: Running coprocessor pre-open hook at 1731769215129Writing region info on filesystem at 1731769215129Initializing all the Stores at 1731769215131 (+2 ms)Instantiating store for column family {NAME => 'colfamily11', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731769215131Instantiating store for column family {NAME => 'colfamily21', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731769215131Instantiating store for column family {NAME => 'colfamily31', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731769215131Obtaining lock to block concurrent updates at 1731769215147 (+16 ms)Preparing flush snapshotting stores in 7352dcdbf9811dcebc51a714a1de4f55 at 1731769215147Finished memstore snapshotting testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,iii,1731769026392.7352dcdbf9811dcebc51a714a1de4f55., syncing WAL and waiting on mvcc, flushsize=dataSize=33744, getHeapSize=85080, getOffHeapSize=0, getCellsCount=703 at 1731769215147Flushing stores of testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,iii,1731769026392.7352dcdbf9811dcebc51a714a1de4f55. at 1731769215147Flushing 7352dcdbf9811dcebc51a714a1de4f55/colfamily11: creating writer at 1731769215147Flushing 7352dcdbf9811dcebc51a714a1de4f55/colfamily11: appending metadata at 1731769215165 (+18 ms)Flushing 7352dcdbf9811dcebc51a714a1de4f55/colfamily11: closing flushed file at 1731769215165Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@6903407a: reopening flushed file at 1731769215185 (+20 ms)Finished flush of dataSize ~32.95 KB/33744, heapSize ~82.62 KB/84600, currentSize=0 B/0 for 7352dcdbf9811dcebc51a714a1de4f55 in 48ms, sequenceid=4, compaction requested=false; wal=null at 1731769215194 (+9 ms)Cleaning up temporary data from old regions at 1731769215196 (+2 ms)Running coprocessor post-open hooks at 1731769215202 (+6 ms)Region opened successfully at 1731769215202 2024-11-16T15:00:15,203 INFO [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegionServer(2236): Post open deploy tasks for testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,iii,1731769026392.7352dcdbf9811dcebc51a714a1de4f55., pid=76, masterSystemTime=1731769214946 2024-11-16T15:00:15,205 DEBUG [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegionServer(2266): Finished post open deploy task for testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,iii,1731769026392.7352dcdbf9811dcebc51a714a1de4f55. 2024-11-16T15:00:15,205 INFO [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] handler.AssignRegionHandler(153): Opened testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,iii,1731769026392.7352dcdbf9811dcebc51a714a1de4f55. 2024-11-16T15:00:15,205 INFO [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] handler.AssignRegionHandler(132): Open testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,mmm,1731769026392.c2acbc05ae7bb1607a5719b701b40383. 2024-11-16T15:00:15,205 DEBUG [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(7752): Opening region: {ENCODED => c2acbc05ae7bb1607a5719b701b40383, NAME => 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,mmm,1731769026392.c2acbc05ae7bb1607a5719b701b40383.', STARTKEY => 'mmm', ENDKEY => 'nnn'} 2024-11-16T15:00:15,205 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=64 updating hbase:meta row=7352dcdbf9811dcebc51a714a1de4f55, regionState=OPEN, openSeqNum=5, regionLocation=fc3fda29f660,39089,1731769023769 2024-11-16T15:00:15,206 DEBUG [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true c2acbc05ae7bb1607a5719b701b40383 2024-11-16T15:00:15,206 DEBUG [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(898): Instantiated testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,mmm,1731769026392.c2acbc05ae7bb1607a5719b701b40383.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-16T15:00:15,206 DEBUG [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(7794): checking encryption for c2acbc05ae7bb1607a5719b701b40383 2024-11-16T15:00:15,206 DEBUG [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(7797): checking classloading for c2acbc05ae7bb1607a5719b701b40383 2024-11-16T15:00:15,207 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=76, ppid=64, state=RUNNABLE, hasLock=false; OpenRegionProcedure 7352dcdbf9811dcebc51a714a1de4f55, server=fc3fda29f660,39089,1731769023769 because future has completed 2024-11-16T15:00:15,207 INFO [StoreOpener-c2acbc05ae7bb1607a5719b701b40383-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily11 of region c2acbc05ae7bb1607a5719b701b40383 2024-11-16T15:00:15,208 INFO [StoreOpener-c2acbc05ae7bb1607a5719b701b40383-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region c2acbc05ae7bb1607a5719b701b40383 columnFamilyName colfamily11 2024-11-16T15:00:15,208 DEBUG [StoreOpener-c2acbc05ae7bb1607a5719b701b40383-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T15:00:15,209 INFO [StoreOpener-c2acbc05ae7bb1607a5719b701b40383-1 {}] regionserver.HStore(327): Store=c2acbc05ae7bb1607a5719b701b40383/colfamily11, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-16T15:00:15,209 INFO [StoreOpener-c2acbc05ae7bb1607a5719b701b40383-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily21 of region c2acbc05ae7bb1607a5719b701b40383 2024-11-16T15:00:15,210 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=76, resume processing ppid=64 2024-11-16T15:00:15,210 INFO [StoreOpener-c2acbc05ae7bb1607a5719b701b40383-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region c2acbc05ae7bb1607a5719b701b40383 columnFamilyName colfamily21 2024-11-16T15:00:15,210 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=76, ppid=64, state=SUCCESS, hasLock=false; OpenRegionProcedure 7352dcdbf9811dcebc51a714a1de4f55, server=fc3fda29f660,39089,1731769023769 in 406 msec 2024-11-16T15:00:15,210 DEBUG [StoreOpener-c2acbc05ae7bb1607a5719b701b40383-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T15:00:15,211 INFO [StoreOpener-c2acbc05ae7bb1607a5719b701b40383-1 {}] regionserver.HStore(327): Store=c2acbc05ae7bb1607a5719b701b40383/colfamily21, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-16T15:00:15,211 INFO [StoreOpener-c2acbc05ae7bb1607a5719b701b40383-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily31 of region c2acbc05ae7bb1607a5719b701b40383 2024-11-16T15:00:15,212 INFO [StoreOpener-c2acbc05ae7bb1607a5719b701b40383-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region c2acbc05ae7bb1607a5719b701b40383 columnFamilyName colfamily31 2024-11-16T15:00:15,212 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=64, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=7352dcdbf9811dcebc51a714a1de4f55, ASSIGN in 581 msec 2024-11-16T15:00:15,212 DEBUG [StoreOpener-c2acbc05ae7bb1607a5719b701b40383-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T15:00:15,212 INFO [StoreOpener-c2acbc05ae7bb1607a5719b701b40383-1 {}] regionserver.HStore(327): Store=c2acbc05ae7bb1607a5719b701b40383/colfamily31, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-16T15:00:15,213 DEBUG [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(1038): replaying wal for c2acbc05ae7bb1607a5719b701b40383 2024-11-16T15:00:15,213 DEBUG [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42223/user/jenkins/test-data/a8176cb1-2ebb-b530-b3c7-53c9e53c186e/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/c2acbc05ae7bb1607a5719b701b40383 2024-11-16T15:00:15,215 DEBUG [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:42223/user/jenkins/test-data/a8176cb1-2ebb-b530-b3c7-53c9e53c186e/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/c2acbc05ae7bb1607a5719b701b40383 2024-11-16T15:00:15,215 INFO [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:42223/user/jenkins/test-data/a8176cb1-2ebb-b530-b3c7-53c9e53c186e/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/c2acbc05ae7bb1607a5719b701b40383/recovered.edits/0000000000000000004 2024-11-16T15:00:15,220 DEBUG [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(5793): Applied 703, skipped 0, firstSequenceIdInLog=4, maxSequenceIdInLog=4, path=hdfs://localhost:42223/user/jenkins/test-data/a8176cb1-2ebb-b530-b3c7-53c9e53c186e/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/c2acbc05ae7bb1607a5719b701b40383/recovered.edits/0000000000000000004 2024-11-16T15:00:15,221 INFO [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(2902): Flushing c2acbc05ae7bb1607a5719b701b40383 3/3 column families, dataSize=32.95 KB heapSize=83.13 KB 2024-11-16T15:00:15,238 DEBUG [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42223/user/jenkins/test-data/a8176cb1-2ebb-b530-b3c7-53c9e53c186e/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/c2acbc05ae7bb1607a5719b701b40383/.tmp/colfamily11/85d5d32b7eb04547a1ff0ee53cf0f32d is 52, key is mmm/colfamily11:colfamily11/1731769030445/Put/seqid=0 2024-11-16T15:00:15,244 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40641 is added to blk_1073741884_1060 (size=39263) 2024-11-16T15:00:15,245 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36531 is added to blk_1073741884_1060 (size=39263) 2024-11-16T15:00:15,245 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41671 is added to blk_1073741884_1060 (size=39263) 2024-11-16T15:00:15,245 INFO [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=32.95 KB at sequenceid=4 (bloomFilter=false), to=hdfs://localhost:42223/user/jenkins/test-data/a8176cb1-2ebb-b530-b3c7-53c9e53c186e/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/c2acbc05ae7bb1607a5719b701b40383/.tmp/colfamily11/85d5d32b7eb04547a1ff0ee53cf0f32d 2024-11-16T15:00:15,252 DEBUG [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42223/user/jenkins/test-data/a8176cb1-2ebb-b530-b3c7-53c9e53c186e/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/c2acbc05ae7bb1607a5719b701b40383/.tmp/colfamily11/85d5d32b7eb04547a1ff0ee53cf0f32d as hdfs://localhost:42223/user/jenkins/test-data/a8176cb1-2ebb-b530-b3c7-53c9e53c186e/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/c2acbc05ae7bb1607a5719b701b40383/colfamily11/85d5d32b7eb04547a1ff0ee53cf0f32d 2024-11-16T15:00:15,258 INFO [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42223/user/jenkins/test-data/a8176cb1-2ebb-b530-b3c7-53c9e53c186e/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/c2acbc05ae7bb1607a5719b701b40383/colfamily11/85d5d32b7eb04547a1ff0ee53cf0f32d, entries=703, sequenceid=4, filesize=38.3 K 2024-11-16T15:00:15,258 INFO [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(3140): Finished flush of dataSize ~32.95 KB/33744, heapSize ~82.62 KB/84600, currentSize=0 B/0 for c2acbc05ae7bb1607a5719b701b40383 in 38ms, sequenceid=4, compaction requested=false; wal=null 2024-11-16T15:00:15,259 DEBUG [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:42223/user/jenkins/test-data/a8176cb1-2ebb-b530-b3c7-53c9e53c186e/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/c2acbc05ae7bb1607a5719b701b40383/recovered.edits/0000000000000000004 2024-11-16T15:00:15,260 DEBUG [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(1048): stopping wal replay for c2acbc05ae7bb1607a5719b701b40383 2024-11-16T15:00:15,260 DEBUG [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(1060): Cleaning up temporary data for c2acbc05ae7bb1607a5719b701b40383 2024-11-16T15:00:15,260 DEBUG [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-16T15:00:15,262 DEBUG [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(1093): writing seq id for c2acbc05ae7bb1607a5719b701b40383 2024-11-16T15:00:15,264 DEBUG [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:42223/user/jenkins/test-data/a8176cb1-2ebb-b530-b3c7-53c9e53c186e/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/c2acbc05ae7bb1607a5719b701b40383/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-16T15:00:15,265 INFO [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(1114): Opened c2acbc05ae7bb1607a5719b701b40383; next sequenceid=5; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=63140017, jitterRate=-0.05914042890071869}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-16T15:00:15,265 DEBUG [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(1122): Running coprocessor post-open hooks for c2acbc05ae7bb1607a5719b701b40383 2024-11-16T15:00:15,265 DEBUG [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(1006): Region open journal for c2acbc05ae7bb1607a5719b701b40383: Running coprocessor pre-open hook at 1731769215206Writing region info on filesystem at 1731769215206Initializing all the Stores at 1731769215207 (+1 ms)Instantiating store for column family {NAME => 'colfamily11', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731769215207Instantiating store for column family {NAME => 'colfamily21', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731769215207Instantiating store for column family {NAME => 'colfamily31', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731769215207Obtaining lock to block concurrent updates at 1731769215221 (+14 ms)Preparing flush snapshotting stores in c2acbc05ae7bb1607a5719b701b40383 at 1731769215221Finished memstore snapshotting testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,mmm,1731769026392.c2acbc05ae7bb1607a5719b701b40383., syncing WAL and waiting on mvcc, flushsize=dataSize=33744, getHeapSize=85080, getOffHeapSize=0, getCellsCount=703 at 1731769215221Flushing stores of testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,mmm,1731769026392.c2acbc05ae7bb1607a5719b701b40383. at 1731769215221Flushing c2acbc05ae7bb1607a5719b701b40383/colfamily11: creating writer at 1731769215221Flushing c2acbc05ae7bb1607a5719b701b40383/colfamily11: appending metadata at 1731769215237 (+16 ms)Flushing c2acbc05ae7bb1607a5719b701b40383/colfamily11: closing flushed file at 1731769215237Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@419c133d: reopening flushed file at 1731769215251 (+14 ms)Finished flush of dataSize ~32.95 KB/33744, heapSize ~82.62 KB/84600, currentSize=0 B/0 for c2acbc05ae7bb1607a5719b701b40383 in 38ms, sequenceid=4, compaction requested=false; wal=null at 1731769215258 (+7 ms)Cleaning up temporary data from old regions at 1731769215260 (+2 ms)Running coprocessor post-open hooks at 1731769215265 (+5 ms)Region opened successfully at 1731769215265 2024-11-16T15:00:15,266 INFO [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegionServer(2236): Post open deploy tasks for testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,mmm,1731769026392.c2acbc05ae7bb1607a5719b701b40383., pid=77, masterSystemTime=1731769214946 2024-11-16T15:00:15,268 DEBUG [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegionServer(2266): Finished post open deploy task for testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,mmm,1731769026392.c2acbc05ae7bb1607a5719b701b40383. 2024-11-16T15:00:15,268 INFO [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] handler.AssignRegionHandler(153): Opened testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,mmm,1731769026392.c2acbc05ae7bb1607a5719b701b40383. 2024-11-16T15:00:15,268 INFO [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] handler.AssignRegionHandler(132): Open testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,sss,1731769026392.33054df03c1be51dc860405880be6075. 2024-11-16T15:00:15,268 DEBUG [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegion(7752): Opening region: {ENCODED => 33054df03c1be51dc860405880be6075, NAME => 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,sss,1731769026392.33054df03c1be51dc860405880be6075.', STARTKEY => 'sss', ENDKEY => 'ttt'} 2024-11-16T15:00:15,269 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=72 updating hbase:meta row=c2acbc05ae7bb1607a5719b701b40383, regionState=OPEN, openSeqNum=5, regionLocation=fc3fda29f660,39089,1731769023769 2024-11-16T15:00:15,269 DEBUG [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true 33054df03c1be51dc860405880be6075 2024-11-16T15:00:15,269 DEBUG [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegion(898): Instantiated testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,sss,1731769026392.33054df03c1be51dc860405880be6075.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-16T15:00:15,269 DEBUG [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegion(7794): checking encryption for 33054df03c1be51dc860405880be6075 2024-11-16T15:00:15,269 DEBUG [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegion(7797): checking classloading for 33054df03c1be51dc860405880be6075 2024-11-16T15:00:15,270 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=77, ppid=72, state=RUNNABLE, hasLock=false; OpenRegionProcedure c2acbc05ae7bb1607a5719b701b40383, server=fc3fda29f660,39089,1731769023769 because future has completed 2024-11-16T15:00:15,271 INFO [StoreOpener-33054df03c1be51dc860405880be6075-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily11 of region 33054df03c1be51dc860405880be6075 2024-11-16T15:00:15,272 INFO [StoreOpener-33054df03c1be51dc860405880be6075-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 33054df03c1be51dc860405880be6075 columnFamilyName colfamily11 2024-11-16T15:00:15,272 DEBUG [StoreOpener-33054df03c1be51dc860405880be6075-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T15:00:15,273 INFO [StoreOpener-33054df03c1be51dc860405880be6075-1 {}] regionserver.HStore(327): Store=33054df03c1be51dc860405880be6075/colfamily11, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-16T15:00:15,273 INFO [StoreOpener-33054df03c1be51dc860405880be6075-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily21 of region 33054df03c1be51dc860405880be6075 2024-11-16T15:00:15,274 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=77, resume processing ppid=72 2024-11-16T15:00:15,274 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=77, ppid=72, state=SUCCESS, hasLock=false; OpenRegionProcedure c2acbc05ae7bb1607a5719b701b40383, server=fc3fda29f660,39089,1731769023769 in 467 msec 2024-11-16T15:00:15,274 INFO [StoreOpener-33054df03c1be51dc860405880be6075-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 33054df03c1be51dc860405880be6075 columnFamilyName colfamily21 2024-11-16T15:00:15,274 DEBUG [StoreOpener-33054df03c1be51dc860405880be6075-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T15:00:15,275 INFO [StoreOpener-33054df03c1be51dc860405880be6075-1 {}] regionserver.HStore(327): Store=33054df03c1be51dc860405880be6075/colfamily21, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-16T15:00:15,275 INFO [StoreOpener-33054df03c1be51dc860405880be6075-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily31 of region 33054df03c1be51dc860405880be6075 2024-11-16T15:00:15,276 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=72, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=c2acbc05ae7bb1607a5719b701b40383, ASSIGN in 645 msec 2024-11-16T15:00:15,276 INFO [StoreOpener-33054df03c1be51dc860405880be6075-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 33054df03c1be51dc860405880be6075 columnFamilyName colfamily31 2024-11-16T15:00:15,276 DEBUG [StoreOpener-33054df03c1be51dc860405880be6075-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T15:00:15,276 INFO [StoreOpener-33054df03c1be51dc860405880be6075-1 {}] regionserver.HStore(327): Store=33054df03c1be51dc860405880be6075/colfamily31, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-16T15:00:15,277 DEBUG [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegion(1038): replaying wal for 33054df03c1be51dc860405880be6075 2024-11-16T15:00:15,277 DEBUG [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42223/user/jenkins/test-data/a8176cb1-2ebb-b530-b3c7-53c9e53c186e/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/33054df03c1be51dc860405880be6075 2024-11-16T15:00:15,279 DEBUG [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:42223/user/jenkins/test-data/a8176cb1-2ebb-b530-b3c7-53c9e53c186e/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/33054df03c1be51dc860405880be6075 2024-11-16T15:00:15,280 INFO [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:42223/user/jenkins/test-data/a8176cb1-2ebb-b530-b3c7-53c9e53c186e/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/33054df03c1be51dc860405880be6075/recovered.edits/0000000000000000004 2024-11-16T15:00:15,285 DEBUG [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegion(5793): Applied 703, skipped 0, firstSequenceIdInLog=4, maxSequenceIdInLog=4, path=hdfs://localhost:42223/user/jenkins/test-data/a8176cb1-2ebb-b530-b3c7-53c9e53c186e/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/33054df03c1be51dc860405880be6075/recovered.edits/0000000000000000004 2024-11-16T15:00:15,285 INFO [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegion(2902): Flushing 33054df03c1be51dc860405880be6075 3/3 column families, dataSize=32.95 KB heapSize=83.13 KB 2024-11-16T15:00:15,302 DEBUG [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42223/user/jenkins/test-data/a8176cb1-2ebb-b530-b3c7-53c9e53c186e/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/33054df03c1be51dc860405880be6075/.tmp/colfamily11/25f93d23f47c400caea8d3e6d84d9fbf is 52, key is sss/colfamily11:colfamily11/1731769030504/Put/seqid=0 2024-11-16T15:00:15,311 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41671 is added to blk_1073741885_1061 (size=39263) 2024-11-16T15:00:15,311 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40641 is added to blk_1073741885_1061 (size=39263) 2024-11-16T15:00:15,312 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36531 is added to blk_1073741885_1061 (size=39263) 2024-11-16T15:00:15,313 INFO [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=32.95 KB at sequenceid=4 (bloomFilter=false), to=hdfs://localhost:42223/user/jenkins/test-data/a8176cb1-2ebb-b530-b3c7-53c9e53c186e/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/33054df03c1be51dc860405880be6075/.tmp/colfamily11/25f93d23f47c400caea8d3e6d84d9fbf 2024-11-16T15:00:15,326 DEBUG [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42223/user/jenkins/test-data/a8176cb1-2ebb-b530-b3c7-53c9e53c186e/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/33054df03c1be51dc860405880be6075/.tmp/colfamily11/25f93d23f47c400caea8d3e6d84d9fbf as hdfs://localhost:42223/user/jenkins/test-data/a8176cb1-2ebb-b530-b3c7-53c9e53c186e/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/33054df03c1be51dc860405880be6075/colfamily11/25f93d23f47c400caea8d3e6d84d9fbf 2024-11-16T15:00:15,333 INFO [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42223/user/jenkins/test-data/a8176cb1-2ebb-b530-b3c7-53c9e53c186e/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/33054df03c1be51dc860405880be6075/colfamily11/25f93d23f47c400caea8d3e6d84d9fbf, entries=703, sequenceid=4, filesize=38.3 K 2024-11-16T15:00:15,334 INFO [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegion(3140): Finished flush of dataSize ~32.95 KB/33744, heapSize ~82.62 KB/84600, currentSize=0 B/0 for 33054df03c1be51dc860405880be6075 in 48ms, sequenceid=4, compaction requested=false; wal=null 2024-11-16T15:00:15,335 DEBUG [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:42223/user/jenkins/test-data/a8176cb1-2ebb-b530-b3c7-53c9e53c186e/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/33054df03c1be51dc860405880be6075/recovered.edits/0000000000000000004 2024-11-16T15:00:15,336 DEBUG [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegion(1048): stopping wal replay for 33054df03c1be51dc860405880be6075 2024-11-16T15:00:15,336 DEBUG [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegion(1060): Cleaning up temporary data for 33054df03c1be51dc860405880be6075 2024-11-16T15:00:15,337 DEBUG [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-16T15:00:15,338 DEBUG [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegion(1093): writing seq id for 33054df03c1be51dc860405880be6075 2024-11-16T15:00:15,341 DEBUG [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:42223/user/jenkins/test-data/a8176cb1-2ebb-b530-b3c7-53c9e53c186e/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/33054df03c1be51dc860405880be6075/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-16T15:00:15,342 INFO [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegion(1114): Opened 33054df03c1be51dc860405880be6075; next sequenceid=5; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=59863114, jitterRate=-0.10797008872032166}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-16T15:00:15,342 DEBUG [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 33054df03c1be51dc860405880be6075 2024-11-16T15:00:15,342 DEBUG [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegion(1006): Region open journal for 33054df03c1be51dc860405880be6075: Running coprocessor pre-open hook at 1731769215269Writing region info on filesystem at 1731769215269Initializing all the Stores at 1731769215271 (+2 ms)Instantiating store for column family {NAME => 'colfamily11', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731769215271Instantiating store for column family {NAME => 'colfamily21', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731769215271Instantiating store for column family {NAME => 'colfamily31', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731769215271Obtaining lock to block concurrent updates at 1731769215285 (+14 ms)Preparing flush snapshotting stores in 33054df03c1be51dc860405880be6075 at 1731769215285Finished memstore snapshotting testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,sss,1731769026392.33054df03c1be51dc860405880be6075., syncing WAL and waiting on mvcc, flushsize=dataSize=33744, getHeapSize=85080, getOffHeapSize=0, getCellsCount=703 at 1731769215285Flushing stores of testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,sss,1731769026392.33054df03c1be51dc860405880be6075. at 1731769215285Flushing 33054df03c1be51dc860405880be6075/colfamily11: creating writer at 1731769215285Flushing 33054df03c1be51dc860405880be6075/colfamily11: appending metadata at 1731769215302 (+17 ms)Flushing 33054df03c1be51dc860405880be6075/colfamily11: closing flushed file at 1731769215302Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@98c9e9d: reopening flushed file at 1731769215325 (+23 ms)Finished flush of dataSize ~32.95 KB/33744, heapSize ~82.62 KB/84600, currentSize=0 B/0 for 33054df03c1be51dc860405880be6075 in 48ms, sequenceid=4, compaction requested=false; wal=null at 1731769215334 (+9 ms)Cleaning up temporary data from old regions at 1731769215336 (+2 ms)Running coprocessor post-open hooks at 1731769215342 (+6 ms)Region opened successfully at 1731769215342 2024-11-16T15:00:15,343 INFO [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegionServer(2236): Post open deploy tasks for testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,sss,1731769026392.33054df03c1be51dc860405880be6075., pid=78, masterSystemTime=1731769214946 2024-11-16T15:00:15,344 DEBUG [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegionServer(2266): Finished post open deploy task for testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,sss,1731769026392.33054df03c1be51dc860405880be6075. 2024-11-16T15:00:15,344 INFO [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] handler.AssignRegionHandler(153): Opened testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,sss,1731769026392.33054df03c1be51dc860405880be6075. 2024-11-16T15:00:15,345 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=69 updating hbase:meta row=33054df03c1be51dc860405880be6075, regionState=OPEN, openSeqNum=5, regionLocation=fc3fda29f660,39089,1731769023769 2024-11-16T15:00:15,347 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=78, ppid=69, state=RUNNABLE, hasLock=false; OpenRegionProcedure 33054df03c1be51dc860405880be6075, server=fc3fda29f660,39089,1731769023769 because future has completed 2024-11-16T15:00:15,349 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=78, resume processing ppid=69 2024-11-16T15:00:15,349 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=78, ppid=69, state=SUCCESS, hasLock=false; OpenRegionProcedure 33054df03c1be51dc860405880be6075, server=fc3fda29f660,39089,1731769023769 in 542 msec 2024-11-16T15:00:15,351 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=69, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=33054df03c1be51dc860405880be6075, ASSIGN in 720 msec 2024-11-16T15:00:15,579 INFO [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=32.95 KB at sequenceid=4 (bloomFilter=false), to=hdfs://localhost:42223/user/jenkins/test-data/a8176cb1-2ebb-b530-b3c7-53c9e53c186e/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/fd3944fc2b78246a853fb2232c747c35/.tmp/colfamily11/1fb29abe9e494d2b965ebb2ab1f0413c 2024-11-16T15:00:15,591 DEBUG [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42223/user/jenkins/test-data/a8176cb1-2ebb-b530-b3c7-53c9e53c186e/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/fd3944fc2b78246a853fb2232c747c35/.tmp/colfamily11/1fb29abe9e494d2b965ebb2ab1f0413c as hdfs://localhost:42223/user/jenkins/test-data/a8176cb1-2ebb-b530-b3c7-53c9e53c186e/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/fd3944fc2b78246a853fb2232c747c35/colfamily11/1fb29abe9e494d2b965ebb2ab1f0413c 2024-11-16T15:00:15,598 INFO [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42223/user/jenkins/test-data/a8176cb1-2ebb-b530-b3c7-53c9e53c186e/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/fd3944fc2b78246a853fb2232c747c35/colfamily11/1fb29abe9e494d2b965ebb2ab1f0413c, entries=703, sequenceid=4, filesize=38.3 K 2024-11-16T15:00:15,599 INFO [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.HRegion(3140): Finished flush of dataSize ~32.95 KB/33744, heapSize ~82.62 KB/84600, currentSize=0 B/0 for fd3944fc2b78246a853fb2232c747c35 in 452ms, sequenceid=4, compaction requested=false; wal=null 2024-11-16T15:00:15,600 DEBUG [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:42223/user/jenkins/test-data/a8176cb1-2ebb-b530-b3c7-53c9e53c186e/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/fd3944fc2b78246a853fb2232c747c35/recovered.edits/0000000000000000004 2024-11-16T15:00:15,601 DEBUG [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.HRegion(1048): stopping wal replay for fd3944fc2b78246a853fb2232c747c35 2024-11-16T15:00:15,601 DEBUG [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.HRegion(1060): Cleaning up temporary data for fd3944fc2b78246a853fb2232c747c35 2024-11-16T15:00:15,602 DEBUG [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-16T15:00:15,603 DEBUG [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.HRegion(1093): writing seq id for fd3944fc2b78246a853fb2232c747c35 2024-11-16T15:00:15,605 DEBUG [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:42223/user/jenkins/test-data/a8176cb1-2ebb-b530-b3c7-53c9e53c186e/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/fd3944fc2b78246a853fb2232c747c35/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-16T15:00:15,606 INFO [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.HRegion(1114): Opened fd3944fc2b78246a853fb2232c747c35; next sequenceid=5; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=68728959, jitterRate=0.02414129674434662}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-16T15:00:15,607 DEBUG [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.HRegion(1122): Running coprocessor post-open hooks for fd3944fc2b78246a853fb2232c747c35 2024-11-16T15:00:15,607 DEBUG [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.HRegion(1006): Region open journal for fd3944fc2b78246a853fb2232c747c35: Running coprocessor pre-open hook at 1731769215129Writing region info on filesystem at 1731769215129Initializing all the Stores at 1731769215131 (+2 ms)Instantiating store for column family {NAME => 'colfamily11', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731769215131Instantiating store for column family {NAME => 'colfamily21', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731769215131Instantiating store for column family {NAME => 'colfamily31', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731769215131Obtaining lock to block concurrent updates at 1731769215148 (+17 ms)Preparing flush snapshotting stores in fd3944fc2b78246a853fb2232c747c35 at 1731769215148Finished memstore snapshotting testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,eee,1731769026392.fd3944fc2b78246a853fb2232c747c35., syncing WAL and waiting on mvcc, flushsize=dataSize=33744, getHeapSize=85080, getOffHeapSize=0, getCellsCount=703 at 1731769215148Flushing stores of testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,eee,1731769026392.fd3944fc2b78246a853fb2232c747c35. at 1731769215148Flushing fd3944fc2b78246a853fb2232c747c35/colfamily11: creating writer at 1731769215148Flushing fd3944fc2b78246a853fb2232c747c35/colfamily11: appending metadata at 1731769215164 (+16 ms)Flushing fd3944fc2b78246a853fb2232c747c35/colfamily11: closing flushed file at 1731769215164Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@d8dccaf: reopening flushed file at 1731769215590 (+426 ms)Finished flush of dataSize ~32.95 KB/33744, heapSize ~82.62 KB/84600, currentSize=0 B/0 for fd3944fc2b78246a853fb2232c747c35 in 452ms, sequenceid=4, compaction requested=false; wal=null at 1731769215599 (+9 ms)Cleaning up temporary data from old regions at 1731769215601 (+2 ms)Running coprocessor post-open hooks at 1731769215607 (+6 ms)Region opened successfully at 1731769215607 2024-11-16T15:00:15,608 INFO [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.HRegionServer(2236): Post open deploy tasks for testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,eee,1731769026392.fd3944fc2b78246a853fb2232c747c35., pid=80, masterSystemTime=1731769214951 2024-11-16T15:00:15,609 DEBUG [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.HRegionServer(2266): Finished post open deploy task for testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,eee,1731769026392.fd3944fc2b78246a853fb2232c747c35. 2024-11-16T15:00:15,610 INFO [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] handler.AssignRegionHandler(153): Opened testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,eee,1731769026392.fd3944fc2b78246a853fb2232c747c35. 2024-11-16T15:00:15,610 INFO [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] handler.AssignRegionHandler(132): Open testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,jjj,1731769026392.7eb7b75d28b808c5c3f2f96e407b4347. 2024-11-16T15:00:15,610 DEBUG [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.HRegion(7752): Opening region: {ENCODED => 7eb7b75d28b808c5c3f2f96e407b4347, NAME => 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,jjj,1731769026392.7eb7b75d28b808c5c3f2f96e407b4347.', STARTKEY => 'jjj', ENDKEY => 'kkk'} 2024-11-16T15:00:15,610 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=67 updating hbase:meta row=fd3944fc2b78246a853fb2232c747c35, regionState=OPEN, openSeqNum=5, regionLocation=fc3fda29f660,42985,1731769023951 2024-11-16T15:00:15,610 DEBUG [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true 7eb7b75d28b808c5c3f2f96e407b4347 2024-11-16T15:00:15,610 DEBUG [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.HRegion(898): Instantiated testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,jjj,1731769026392.7eb7b75d28b808c5c3f2f96e407b4347.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-16T15:00:15,610 DEBUG [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.HRegion(7794): checking encryption for 7eb7b75d28b808c5c3f2f96e407b4347 2024-11-16T15:00:15,610 DEBUG [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.HRegion(7797): checking classloading for 7eb7b75d28b808c5c3f2f96e407b4347 2024-11-16T15:00:15,612 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=80, ppid=67, state=RUNNABLE, hasLock=false; OpenRegionProcedure fd3944fc2b78246a853fb2232c747c35, server=fc3fda29f660,42985,1731769023951 because future has completed 2024-11-16T15:00:15,612 INFO [StoreOpener-7eb7b75d28b808c5c3f2f96e407b4347-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily11 of region 7eb7b75d28b808c5c3f2f96e407b4347 2024-11-16T15:00:15,613 INFO [StoreOpener-7eb7b75d28b808c5c3f2f96e407b4347-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 7eb7b75d28b808c5c3f2f96e407b4347 columnFamilyName colfamily11 2024-11-16T15:00:15,613 DEBUG [StoreOpener-7eb7b75d28b808c5c3f2f96e407b4347-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T15:00:15,613 INFO [StoreOpener-7eb7b75d28b808c5c3f2f96e407b4347-1 {}] regionserver.HStore(327): Store=7eb7b75d28b808c5c3f2f96e407b4347/colfamily11, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-16T15:00:15,613 INFO [StoreOpener-7eb7b75d28b808c5c3f2f96e407b4347-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily21 of region 7eb7b75d28b808c5c3f2f96e407b4347 2024-11-16T15:00:15,614 INFO [StoreOpener-7eb7b75d28b808c5c3f2f96e407b4347-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 7eb7b75d28b808c5c3f2f96e407b4347 columnFamilyName colfamily21 2024-11-16T15:00:15,614 DEBUG [StoreOpener-7eb7b75d28b808c5c3f2f96e407b4347-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T15:00:15,614 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=80, resume processing ppid=67 2024-11-16T15:00:15,614 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=80, ppid=67, state=SUCCESS, hasLock=false; OpenRegionProcedure fd3944fc2b78246a853fb2232c747c35, server=fc3fda29f660,42985,1731769023951 in 791 msec 2024-11-16T15:00:15,614 INFO [StoreOpener-7eb7b75d28b808c5c3f2f96e407b4347-1 {}] regionserver.HStore(327): Store=7eb7b75d28b808c5c3f2f96e407b4347/colfamily21, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-16T15:00:15,615 INFO [StoreOpener-7eb7b75d28b808c5c3f2f96e407b4347-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily31 of region 7eb7b75d28b808c5c3f2f96e407b4347 2024-11-16T15:00:15,615 INFO [StoreOpener-7eb7b75d28b808c5c3f2f96e407b4347-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 7eb7b75d28b808c5c3f2f96e407b4347 columnFamilyName colfamily31 2024-11-16T15:00:15,615 DEBUG [StoreOpener-7eb7b75d28b808c5c3f2f96e407b4347-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T15:00:15,616 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=67, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=fd3944fc2b78246a853fb2232c747c35, ASSIGN in 986 msec 2024-11-16T15:00:15,616 INFO [StoreOpener-7eb7b75d28b808c5c3f2f96e407b4347-1 {}] regionserver.HStore(327): Store=7eb7b75d28b808c5c3f2f96e407b4347/colfamily31, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-16T15:00:15,616 DEBUG [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.HRegion(1038): replaying wal for 7eb7b75d28b808c5c3f2f96e407b4347 2024-11-16T15:00:15,617 DEBUG [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42223/user/jenkins/test-data/a8176cb1-2ebb-b530-b3c7-53c9e53c186e/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/7eb7b75d28b808c5c3f2f96e407b4347 2024-11-16T15:00:15,618 DEBUG [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:42223/user/jenkins/test-data/a8176cb1-2ebb-b530-b3c7-53c9e53c186e/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/7eb7b75d28b808c5c3f2f96e407b4347 2024-11-16T15:00:15,619 INFO [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:42223/user/jenkins/test-data/a8176cb1-2ebb-b530-b3c7-53c9e53c186e/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/7eb7b75d28b808c5c3f2f96e407b4347/recovered.edits/0000000000000000004 2024-11-16T15:00:15,623 DEBUG [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.HRegion(5793): Applied 703, skipped 0, firstSequenceIdInLog=4, maxSequenceIdInLog=4, path=hdfs://localhost:42223/user/jenkins/test-data/a8176cb1-2ebb-b530-b3c7-53c9e53c186e/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/7eb7b75d28b808c5c3f2f96e407b4347/recovered.edits/0000000000000000004 2024-11-16T15:00:15,623 INFO [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.HRegion(2902): Flushing 7eb7b75d28b808c5c3f2f96e407b4347 3/3 column families, dataSize=32.95 KB heapSize=83.13 KB 2024-11-16T15:00:15,638 DEBUG [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42223/user/jenkins/test-data/a8176cb1-2ebb-b530-b3c7-53c9e53c186e/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/7eb7b75d28b808c5c3f2f96e407b4347/.tmp/colfamily11/f120bcc3ef20465fa1f7339e97ed840d is 52, key is jjj/colfamily11:colfamily11/1731769030410/Put/seqid=0 2024-11-16T15:00:15,644 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36531 is added to blk_1073741886_1062 (size=39263) 2024-11-16T15:00:15,644 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40641 is added to blk_1073741886_1062 (size=39263) 2024-11-16T15:00:15,645 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41671 is added to blk_1073741886_1062 (size=39263) 2024-11-16T15:00:15,645 INFO [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=32.95 KB at sequenceid=4 (bloomFilter=false), to=hdfs://localhost:42223/user/jenkins/test-data/a8176cb1-2ebb-b530-b3c7-53c9e53c186e/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/7eb7b75d28b808c5c3f2f96e407b4347/.tmp/colfamily11/f120bcc3ef20465fa1f7339e97ed840d 2024-11-16T15:00:15,652 DEBUG [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42223/user/jenkins/test-data/a8176cb1-2ebb-b530-b3c7-53c9e53c186e/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/7eb7b75d28b808c5c3f2f96e407b4347/.tmp/colfamily11/f120bcc3ef20465fa1f7339e97ed840d as hdfs://localhost:42223/user/jenkins/test-data/a8176cb1-2ebb-b530-b3c7-53c9e53c186e/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/7eb7b75d28b808c5c3f2f96e407b4347/colfamily11/f120bcc3ef20465fa1f7339e97ed840d 2024-11-16T15:00:15,658 INFO [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42223/user/jenkins/test-data/a8176cb1-2ebb-b530-b3c7-53c9e53c186e/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/7eb7b75d28b808c5c3f2f96e407b4347/colfamily11/f120bcc3ef20465fa1f7339e97ed840d, entries=703, sequenceid=4, filesize=38.3 K 2024-11-16T15:00:15,659 INFO [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.HRegion(3140): Finished flush of dataSize ~32.95 KB/33744, heapSize ~82.62 KB/84600, currentSize=0 B/0 for 7eb7b75d28b808c5c3f2f96e407b4347 in 35ms, sequenceid=4, compaction requested=false; wal=null 2024-11-16T15:00:15,659 DEBUG [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:42223/user/jenkins/test-data/a8176cb1-2ebb-b530-b3c7-53c9e53c186e/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/7eb7b75d28b808c5c3f2f96e407b4347/recovered.edits/0000000000000000004 2024-11-16T15:00:15,660 DEBUG [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.HRegion(1048): stopping wal replay for 7eb7b75d28b808c5c3f2f96e407b4347 2024-11-16T15:00:15,660 DEBUG [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.HRegion(1060): Cleaning up temporary data for 7eb7b75d28b808c5c3f2f96e407b4347 2024-11-16T15:00:15,661 DEBUG [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-16T15:00:15,662 DEBUG [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.HRegion(1093): writing seq id for 7eb7b75d28b808c5c3f2f96e407b4347 2024-11-16T15:00:15,664 DEBUG [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:42223/user/jenkins/test-data/a8176cb1-2ebb-b530-b3c7-53c9e53c186e/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/7eb7b75d28b808c5c3f2f96e407b4347/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-16T15:00:15,665 INFO [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.HRegion(1114): Opened 7eb7b75d28b808c5c3f2f96e407b4347; next sequenceid=5; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=74145039, jitterRate=0.10484717786312103}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-16T15:00:15,665 DEBUG [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 7eb7b75d28b808c5c3f2f96e407b4347 2024-11-16T15:00:15,665 DEBUG [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.HRegion(1006): Region open journal for 7eb7b75d28b808c5c3f2f96e407b4347: Running coprocessor pre-open hook at 1731769215610Writing region info on filesystem at 1731769215610Initializing all the Stores at 1731769215611 (+1 ms)Instantiating store for column family {NAME => 'colfamily11', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731769215611Instantiating store for column family {NAME => 'colfamily21', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731769215612 (+1 ms)Instantiating store for column family {NAME => 'colfamily31', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731769215612Obtaining lock to block concurrent updates at 1731769215623 (+11 ms)Preparing flush snapshotting stores in 7eb7b75d28b808c5c3f2f96e407b4347 at 1731769215623Finished memstore snapshotting testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,jjj,1731769026392.7eb7b75d28b808c5c3f2f96e407b4347., syncing WAL and waiting on mvcc, flushsize=dataSize=33744, getHeapSize=85080, getOffHeapSize=0, getCellsCount=703 at 1731769215623Flushing stores of testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,jjj,1731769026392.7eb7b75d28b808c5c3f2f96e407b4347. at 1731769215623Flushing 7eb7b75d28b808c5c3f2f96e407b4347/colfamily11: creating writer at 1731769215623Flushing 7eb7b75d28b808c5c3f2f96e407b4347/colfamily11: appending metadata at 1731769215638 (+15 ms)Flushing 7eb7b75d28b808c5c3f2f96e407b4347/colfamily11: closing flushed file at 1731769215638Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@7b93e32: reopening flushed file at 1731769215651 (+13 ms)Finished flush of dataSize ~32.95 KB/33744, heapSize ~82.62 KB/84600, currentSize=0 B/0 for 7eb7b75d28b808c5c3f2f96e407b4347 in 35ms, sequenceid=4, compaction requested=false; wal=null at 1731769215659 (+8 ms)Cleaning up temporary data from old regions at 1731769215660 (+1 ms)Running coprocessor post-open hooks at 1731769215665 (+5 ms)Region opened successfully at 1731769215665 2024-11-16T15:00:15,666 INFO [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.HRegionServer(2236): Post open deploy tasks for testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,jjj,1731769026392.7eb7b75d28b808c5c3f2f96e407b4347., pid=81, masterSystemTime=1731769214951 2024-11-16T15:00:15,667 DEBUG [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.HRegionServer(2266): Finished post open deploy task for testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,jjj,1731769026392.7eb7b75d28b808c5c3f2f96e407b4347. 2024-11-16T15:00:15,668 INFO [RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] handler.AssignRegionHandler(153): Opened testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,jjj,1731769026392.7eb7b75d28b808c5c3f2f96e407b4347. 2024-11-16T15:00:15,668 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=68 updating hbase:meta row=7eb7b75d28b808c5c3f2f96e407b4347, regionState=OPEN, openSeqNum=5, regionLocation=fc3fda29f660,42985,1731769023951 2024-11-16T15:00:15,669 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=81, ppid=68, state=RUNNABLE, hasLock=false; OpenRegionProcedure 7eb7b75d28b808c5c3f2f96e407b4347, server=fc3fda29f660,42985,1731769023951 because future has completed 2024-11-16T15:00:15,672 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=81, resume processing ppid=68 2024-11-16T15:00:15,672 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=81, ppid=68, state=SUCCESS, hasLock=false; OpenRegionProcedure 7eb7b75d28b808c5c3f2f96e407b4347, server=fc3fda29f660,42985,1731769023951 in 846 msec 2024-11-16T15:00:15,674 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=68, resume processing ppid=57 2024-11-16T15:00:15,674 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=68, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=7eb7b75d28b808c5c3f2f96e407b4347, ASSIGN in 1.0430 sec 2024-11-16T15:00:15,674 INFO [PEWorker-1 {}] procedure.ServerCrashProcedure(291): removed crashed server fc3fda29f660,38897,1731769023879 after splitting done 2024-11-16T15:00:15,674 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(132): Toggle KILL before store update to: false 2024-11-16T15:00:15,674 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update: pid=57, state=SUCCESS, hasLock=true; ServerCrashProcedure fc3fda29f660,38897,1731769023879, splitWal=true, meta=true 2024-11-16T15:00:15,674 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-16T15:00:15,674 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update: pid=57, state=SUCCESS, hasLock=true; ServerCrashProcedure fc3fda29f660,38897,1731769023879, splitWal=true, meta=true at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1879) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T15:00:15,880 INFO [Time-limited test {}] procedure.MasterProcedureTestingUtility(473): Restart 8 exec state=pid=57, state=SUCCESS, hasLock=true; ServerCrashProcedure fc3fda29f660,38897,1731769023879, splitWal=true, meta=true 2024-11-16T15:00:15,880 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-16T15:00:15,880 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-16T15:00:15,880 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-16T15:00:15,880 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-16T15:00:15,880 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-16T15:00:15,881 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-16T15:00:15,881 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-16T15:00:15,881 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-16T15:00:15,882 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 1 msec 2024-11-16T15:00:15,885 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=57, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-16T15:00:15,885 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-16T15:00:15,885 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=76, ppid=64, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-16T15:00:15,885 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-16T15:00:15,885 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=79, ppid=65, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-16T15:00:15,885 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-16T15:00:15,885 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=73, ppid=66, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-16T15:00:15,886 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=67, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-16T15:00:15,886 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=80, ppid=67, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-16T15:00:15,886 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=68, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-16T15:00:15,886 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=81, ppid=68, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-16T15:00:15,886 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=69, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-16T15:00:15,886 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=78, ppid=69, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-16T15:00:15,886 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=70, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-16T15:00:15,886 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=75, ppid=70, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-16T15:00:15,886 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=71, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-16T15:00:15,886 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=74, ppid=71, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-16T15:00:15,886 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=72, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-16T15:00:15,886 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=77, ppid=72, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-16T15:00:15,886 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=58, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-16T15:00:15,886 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=59, ppid=58, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-16T15:00:15,886 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-16T15:00:15,887 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-16T15:00:15,887 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-16T15:00:15,887 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=62, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-16T15:00:15,887 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-16T15:00:15,888 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true 2024-11-16T15:00:15,891 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=57, state=RUNNABLE:SERVER_CRASH_FINISH, hasLock=false; ServerCrashProcedure fc3fda29f660,38897,1731769023879, splitWal=true, meta=true 2024-11-16T15:00:15,891 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure fc3fda29f660%2C38897%2C1731769023879.meta.1731769025887.meta, worker=fc3fda29f660,42985,1731769023951 2024-11-16T15:00:15,891 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure fc3fda29f660%2C38897%2C1731769023879.meta.1731769025887.meta, worker=fc3fda29f660,42985,1731769023951 2024-11-16T15:00:15,891 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-16T15:00:15,891 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=fc3fda29f660,39089,1731769023769 2024-11-16T15:00:15,891 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure fc3fda29f660%2C38897%2C1731769023879.1731769025434, worker=fc3fda29f660,39089,1731769023769 2024-11-16T15:00:15,891 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure fc3fda29f660%2C38897%2C1731769023879.1731769025434, worker=fc3fda29f660,39089,1731769023769 2024-11-16T15:00:15,891 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=7352dcdbf9811dcebc51a714a1de4f55, ASSIGN 2024-11-16T15:00:15,891 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=7f5c9df3a544933227f58b052f743f72, ASSIGN 2024-11-16T15:00:15,891 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=426ff231e487ce9f8492776ca1dacc85, ASSIGN 2024-11-16T15:00:15,891 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=67, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=fd3944fc2b78246a853fb2232c747c35, ASSIGN 2024-11-16T15:00:15,891 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=68, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=7eb7b75d28b808c5c3f2f96e407b4347, ASSIGN 2024-11-16T15:00:15,891 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=69, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=33054df03c1be51dc860405880be6075, ASSIGN 2024-11-16T15:00:15,891 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=70, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=b5566c7e10a0e001c7f58316bdd43c2f, ASSIGN 2024-11-16T15:00:15,891 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=71, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=4f0d635e77a1e3dd04c4d107bb0f1564, ASSIGN 2024-11-16T15:00:15,891 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=72, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=c2acbc05ae7bb1607a5719b701b40383, ASSIGN 2024-11-16T15:00:15,891 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=73, ppid=66, state=SUCCESS, hasLock=false; OpenRegionProcedure 426ff231e487ce9f8492776ca1dacc85, server=fc3fda29f660,39089,1731769023769 2024-11-16T15:00:15,891 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=74, ppid=71, state=SUCCESS, hasLock=false; OpenRegionProcedure 4f0d635e77a1e3dd04c4d107bb0f1564, server=fc3fda29f660,39089,1731769023769 2024-11-16T15:00:15,891 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=75, ppid=70, state=SUCCESS, hasLock=false; OpenRegionProcedure b5566c7e10a0e001c7f58316bdd43c2f, server=fc3fda29f660,42985,1731769023951 2024-11-16T15:00:15,891 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=76, ppid=64, state=SUCCESS, hasLock=false; OpenRegionProcedure 7352dcdbf9811dcebc51a714a1de4f55, server=fc3fda29f660,39089,1731769023769 2024-11-16T15:00:15,891 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=77, ppid=72, state=SUCCESS, hasLock=false; OpenRegionProcedure c2acbc05ae7bb1607a5719b701b40383, server=fc3fda29f660,39089,1731769023769 2024-11-16T15:00:15,891 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=78, ppid=69, state=SUCCESS, hasLock=false; OpenRegionProcedure 33054df03c1be51dc860405880be6075, server=fc3fda29f660,39089,1731769023769 2024-11-16T15:00:15,891 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=79, ppid=65, state=SUCCESS, hasLock=false; OpenRegionProcedure 7f5c9df3a544933227f58b052f743f72, server=fc3fda29f660,42985,1731769023951 2024-11-16T15:00:15,891 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=80, ppid=67, state=SUCCESS, hasLock=false; OpenRegionProcedure fd3944fc2b78246a853fb2232c747c35, server=fc3fda29f660,42985,1731769023951 2024-11-16T15:00:15,891 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=81, ppid=68, state=SUCCESS, hasLock=false; OpenRegionProcedure 7eb7b75d28b808c5c3f2f96e407b4347, server=fc3fda29f660,42985,1731769023951 2024-11-16T15:00:15,892 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=57, state=RUNNABLE:SERVER_CRASH_FINISH, hasLock=false; ServerCrashProcedure fc3fda29f660,38897,1731769023879, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-16T15:00:15,892 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure fc3fda29f660%2C38897%2C1731769023879.meta.1731769025887.meta, worker=fc3fda29f660,42985,1731769023951 didn't hold the lock before restarting, skip acquiring lock. 2024-11-16T15:00:15,892 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure fc3fda29f660%2C38897%2C1731769023879.meta.1731769025887.meta, worker=fc3fda29f660,42985,1731769023951 didn't hold the lock before restarting, skip acquiring lock. 2024-11-16T15:00:15,892 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-16T15:00:15,892 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=fc3fda29f660,39089,1731769023769 didn't hold the lock before restarting, skip acquiring lock. 2024-11-16T15:00:15,892 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure fc3fda29f660%2C38897%2C1731769023879.1731769025434, worker=fc3fda29f660,39089,1731769023769 didn't hold the lock before restarting, skip acquiring lock. 2024-11-16T15:00:15,892 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure fc3fda29f660%2C38897%2C1731769023879.1731769025434, worker=fc3fda29f660,39089,1731769023769 didn't hold the lock before restarting, skip acquiring lock. 2024-11-16T15:00:15,892 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=64, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=7352dcdbf9811dcebc51a714a1de4f55, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-16T15:00:15,892 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=65, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=7f5c9df3a544933227f58b052f743f72, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-16T15:00:15,892 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=66, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=426ff231e487ce9f8492776ca1dacc85, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-16T15:00:15,892 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=67, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=fd3944fc2b78246a853fb2232c747c35, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-16T15:00:15,892 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=68, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=7eb7b75d28b808c5c3f2f96e407b4347, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-16T15:00:15,892 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=69, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=33054df03c1be51dc860405880be6075, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-16T15:00:15,892 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=70, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=b5566c7e10a0e001c7f58316bdd43c2f, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-16T15:00:15,892 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=71, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=4f0d635e77a1e3dd04c4d107bb0f1564, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-16T15:00:15,892 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=72, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=c2acbc05ae7bb1607a5719b701b40383, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-16T15:00:15,892 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=73, ppid=66, state=SUCCESS, hasLock=false; OpenRegionProcedure 426ff231e487ce9f8492776ca1dacc85, server=fc3fda29f660,39089,1731769023769 didn't hold the lock before restarting, skip acquiring lock. 2024-11-16T15:00:15,892 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=74, ppid=71, state=SUCCESS, hasLock=false; OpenRegionProcedure 4f0d635e77a1e3dd04c4d107bb0f1564, server=fc3fda29f660,39089,1731769023769 didn't hold the lock before restarting, skip acquiring lock. 2024-11-16T15:00:15,892 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=75, ppid=70, state=SUCCESS, hasLock=false; OpenRegionProcedure b5566c7e10a0e001c7f58316bdd43c2f, server=fc3fda29f660,42985,1731769023951 didn't hold the lock before restarting, skip acquiring lock. 2024-11-16T15:00:15,892 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=76, ppid=64, state=SUCCESS, hasLock=false; OpenRegionProcedure 7352dcdbf9811dcebc51a714a1de4f55, server=fc3fda29f660,39089,1731769023769 didn't hold the lock before restarting, skip acquiring lock. 2024-11-16T15:00:15,892 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=77, ppid=72, state=SUCCESS, hasLock=false; OpenRegionProcedure c2acbc05ae7bb1607a5719b701b40383, server=fc3fda29f660,39089,1731769023769 didn't hold the lock before restarting, skip acquiring lock. 2024-11-16T15:00:15,892 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=78, ppid=69, state=SUCCESS, hasLock=false; OpenRegionProcedure 33054df03c1be51dc860405880be6075, server=fc3fda29f660,39089,1731769023769 didn't hold the lock before restarting, skip acquiring lock. 2024-11-16T15:00:15,892 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=79, ppid=65, state=SUCCESS, hasLock=false; OpenRegionProcedure 7f5c9df3a544933227f58b052f743f72, server=fc3fda29f660,42985,1731769023951 didn't hold the lock before restarting, skip acquiring lock. 2024-11-16T15:00:15,892 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=80, ppid=67, state=SUCCESS, hasLock=false; OpenRegionProcedure fd3944fc2b78246a853fb2232c747c35, server=fc3fda29f660,42985,1731769023951 didn't hold the lock before restarting, skip acquiring lock. 2024-11-16T15:00:15,892 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=81, ppid=68, state=SUCCESS, hasLock=false; OpenRegionProcedure 7eb7b75d28b808c5c3f2f96e407b4347, server=fc3fda29f660,42985,1731769023951 didn't hold the lock before restarting, skip acquiring lock. 2024-11-16T15:00:15,892 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 10 msec 2024-11-16T15:00:15,893 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=fc3fda29f660,39089,1731769023769, regionLocation=fc3fda29f660,39089,1731769023769, openSeqNum=61 2024-11-16T15:00:15,893 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=fc3fda29f660,39089,1731769023769, table=hbase:meta, region=1588230740 2024-11-16T15:00:15,893 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as fc3fda29f660,39089,1731769023769, state=OPEN 2024-11-16T15:00:15,905 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=43615 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:630) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T15:00:15,905 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=43615 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:630) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T15:00:15,936 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42985-0x101456fc9c70003, quorum=127.0.0.1:63224, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-16T15:00:15,936 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43615-0x101456fc9c70000, quorum=127.0.0.1:63224, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-16T15:00:15,936 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39089-0x101456fc9c70001, quorum=127.0.0.1:63224, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-16T15:00:15,937 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-16T15:00:15,937 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-16T15:00:15,937 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-16T15:00:15,938 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731769245938 2024-11-16T15:00:15,938 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-16T15:00:15,951 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4f0d635e77a1e3dd04c4d107bb0f1564, regionState=OPEN, lastHost=fc3fda29f660,39089,1731769023769, regionLocation=fc3fda29f660,39089,1731769023769, openSeqNum=5 2024-11-16T15:00:15,951 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c3dde5f5f668d914608a2987d4f4c001, regionState=OPEN, lastHost=fc3fda29f660,42985,1731769023951, regionLocation=fc3fda29f660,42985,1731769023951, openSeqNum=2 2024-11-16T15:00:15,952 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5bb22144956dcd43c9b1d957232b11e7, regionState=OPEN, lastHost=fc3fda29f660,39089,1731769023769, regionLocation=fc3fda29f660,39089,1731769023769, openSeqNum=2 2024-11-16T15:00:15,952 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=dde7f54607f11c5fb4271e361e82dabd, regionState=OPEN, lastHost=fc3fda29f660,39089,1731769023769, regionLocation=fc3fda29f660,39089,1731769023769, openSeqNum=2 2024-11-16T15:00:15,952 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=fd3944fc2b78246a853fb2232c747c35, regionState=OPEN, lastHost=fc3fda29f660,42985,1731769023951, regionLocation=fc3fda29f660,42985,1731769023951, openSeqNum=5 2024-11-16T15:00:15,952 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a5db9431e22152bd2a82e84504d2879a, regionState=OPEN, lastHost=fc3fda29f660,42985,1731769023951, regionLocation=fc3fda29f660,42985,1731769023951, openSeqNum=2 2024-11-16T15:00:15,952 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=dcd6414ddebad66e87ab37a6dba924f3, regionState=OPEN, lastHost=fc3fda29f660,42985,1731769023951, regionLocation=fc3fda29f660,42985,1731769023951, openSeqNum=2 2024-11-16T15:00:15,952 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1d97df67e5110b0fe1a1e4b7c1341cbb, regionState=OPEN, lastHost=fc3fda29f660,42985,1731769023951, regionLocation=fc3fda29f660,42985,1731769023951, openSeqNum=2 2024-11-16T15:00:15,952 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7352dcdbf9811dcebc51a714a1de4f55, regionState=OPEN, lastHost=fc3fda29f660,39089,1731769023769, regionLocation=fc3fda29f660,39089,1731769023769, openSeqNum=5 2024-11-16T15:00:15,952 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7eb7b75d28b808c5c3f2f96e407b4347, regionState=OPEN, lastHost=fc3fda29f660,42985,1731769023951, regionLocation=fc3fda29f660,42985,1731769023951, openSeqNum=5 2024-11-16T15:00:15,952 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b4f73326a2a6a279d503e557c6ed452f, regionState=OPEN, lastHost=fc3fda29f660,39089,1731769023769, regionLocation=fc3fda29f660,39089,1731769023769, openSeqNum=2 2024-11-16T15:00:15,952 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=266565e60d7419a2a30ceec1b0f6519b, regionState=OPEN, lastHost=fc3fda29f660,42985,1731769023951, regionLocation=fc3fda29f660,42985,1731769023951, openSeqNum=2 2024-11-16T15:00:15,952 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c2acbc05ae7bb1607a5719b701b40383, regionState=OPEN, lastHost=fc3fda29f660,39089,1731769023769, regionLocation=fc3fda29f660,39089,1731769023769, openSeqNum=5 2024-11-16T15:00:15,953 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=994ca1526cf6c4838e65f85444853f1a, regionState=OPEN, lastHost=fc3fda29f660,39089,1731769023769, regionLocation=fc3fda29f660,39089,1731769023769, openSeqNum=2 2024-11-16T15:00:15,953 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=299131947f15f9c44bcfe22cd6623096, regionState=OPEN, lastHost=fc3fda29f660,42985,1731769023951, regionLocation=fc3fda29f660,42985,1731769023951, openSeqNum=2 2024-11-16T15:00:15,953 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7f5c9df3a544933227f58b052f743f72, regionState=OPEN, lastHost=fc3fda29f660,42985,1731769023951, regionLocation=fc3fda29f660,42985,1731769023951, openSeqNum=5 2024-11-16T15:00:15,953 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=18f03e2509a3a89e4a69ba87fad73940, regionState=OPEN, lastHost=fc3fda29f660,39089,1731769023769, regionLocation=fc3fda29f660,39089,1731769023769, openSeqNum=2 2024-11-16T15:00:15,953 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b364a006589fe0d2474ffa45692a6efc, regionState=OPEN, lastHost=fc3fda29f660,42985,1731769023951, regionLocation=fc3fda29f660,42985,1731769023951, openSeqNum=2 2024-11-16T15:00:15,953 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=33054df03c1be51dc860405880be6075, regionState=OPEN, lastHost=fc3fda29f660,39089,1731769023769, regionLocation=fc3fda29f660,39089,1731769023769, openSeqNum=5 2024-11-16T15:00:15,953 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7f11f79823720d664105d8643dfa862d, regionState=OPEN, lastHost=fc3fda29f660,39089,1731769023769, regionLocation=fc3fda29f660,39089,1731769023769, openSeqNum=2 2024-11-16T15:00:15,953 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=35ef9a7f35bb0b08ec1d53b82d3f9777, regionState=OPEN, lastHost=fc3fda29f660,42985,1731769023951, regionLocation=fc3fda29f660,42985,1731769023951, openSeqNum=2 2024-11-16T15:00:15,953 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b5566c7e10a0e001c7f58316bdd43c2f, regionState=OPEN, lastHost=fc3fda29f660,42985,1731769023951, regionLocation=fc3fda29f660,42985,1731769023951, openSeqNum=5 2024-11-16T15:00:15,953 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=62a6d124ad9d161782c0632102c766de, regionState=OPEN, lastHost=fc3fda29f660,39089,1731769023769, regionLocation=fc3fda29f660,39089,1731769023769, openSeqNum=2 2024-11-16T15:00:15,953 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=dc4fcaece350cd66f6bf88cfbdff6f15, regionState=OPEN, lastHost=fc3fda29f660,42985,1731769023951, regionLocation=fc3fda29f660,42985,1731769023951, openSeqNum=2 2024-11-16T15:00:15,954 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=426ff231e487ce9f8492776ca1dacc85, regionState=OPEN, lastHost=fc3fda29f660,39089,1731769023769, regionLocation=fc3fda29f660,39089,1731769023769, openSeqNum=5 2024-11-16T15:00:15,954 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2f41a8c996224c818e917389bd3f5141, regionState=OPEN, lastHost=fc3fda29f660,39089,1731769023769, regionLocation=fc3fda29f660,39089,1731769023769, openSeqNum=2 2024-11-16T15:00:15,954 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-11-16T15:00:15,954 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731769275954 2024-11-16T15:00:15,954 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731769335954 2024-11-16T15:00:15,954 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 15 msec 2024-11-16T15:00:15,954 INFO [PEWorker-1 {}] procedure.ServerCrashProcedure(291): removed crashed server fc3fda29f660,38897,1731769023879 after splitting done 2024-11-16T15:00:15,954 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(132): Toggle KILL before store update to: true 2024-11-16T15:00:15,957 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=57, state=SUCCESS, hasLock=false; ServerCrashProcedure fc3fda29f660,38897,1731769023879, splitWal=true, meta=true in 3 mins, 4.632 sec 2024-11-16T15:00:16,213 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52970, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-16T15:00:16,219 WARN [RPCClient-NioEventLoopGroup-6-3 {}] ipc.NettyRpcConnection$2(409): Exception encountered while connecting to the server fc3fda29f660:38897 org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: Connection refused: fc3fda29f660/172.17.0.2:38897 Caused by: java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hbase.thirdparty.io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(NioSocketChannel.java:336) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:339) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T15:00:16,220 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,,1731769026392.4f0d635e77a1e3dd04c4d107bb0f1564., hostname=fc3fda29f660,38897,1731769023879, seqNum=2 , the old value is region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,,1731769026392.4f0d635e77a1e3dd04c4d107bb0f1564., hostname=fc3fda29f660,38897,1731769023879, seqNum=2, error=java.net.ConnectException: Call to address=fc3fda29f660:38897 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: Connection refused: fc3fda29f660/172.17.0.2:38897 2024-11-16T15:00:16,220 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,,1731769026392.4f0d635e77a1e3dd04c4d107bb0f1564., hostname=fc3fda29f660,38897,1731769023879, seqNum=2 is java.net.ConnectException: Connection refused 2024-11-16T15:00:16,220 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncRegionLocatorHelper(88): Try removing region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,,1731769026392.4f0d635e77a1e3dd04c4d107bb0f1564., hostname=fc3fda29f660,38897,1731769023879, seqNum=2 from cache 2024-11-16T15:00:16,220 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] ipc.FailedServers(52): Added failed server with address fc3fda29f660:38897 to list caused by org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: Connection refused: fc3fda29f660/172.17.0.2:38897 2024-11-16T15:00:16,330 DEBUG [Async-Client-Retry-Timer-pool-0 {}] ipc.AbstractRpcClient(357): Not trying to connect to fc3fda29f660:38897 this server is in the failed servers list 2024-11-16T15:00:16,330 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=fc3fda29f660,38897,1731769023879, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=fc3fda29f660,38897,1731769023879, seqNum=-1, error=org.apache.hadoop.hbase.ipc.FailedServerException: Call to address=fc3fda29f660:38897 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: fc3fda29f660:38897 2024-11-16T15:00:16,330 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:meta,,1.1588230740, hostname=fc3fda29f660,38897,1731769023879, seqNum=-1 is org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: fc3fda29f660:38897 2024-11-16T15:00:16,330 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:meta,,1.1588230740, hostname=fc3fda29f660,38897,1731769023879, seqNum=-1 from cache 2024-11-16T15:00:16,439 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-16T15:00:16,440 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52976, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-16T15:00:16,441 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fc3fda29f660,39089,1731769023769, seqNum=-1] 2024-11-16T15:00:16,444 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51014, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-16T15:00:16,449 DEBUG [RPCClient-NioEventLoopGroup-6-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,,1731769026392.4f0d635e77a1e3dd04c4d107bb0f1564., hostname=fc3fda29f660,39089,1731769023769, seqNum=5] 2024-11-16T15:00:16,463 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60500, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-16T15:00:16,478 DEBUG [RPCClient-NioEventLoopGroup-6-6 {}] ipc.AbstractRpcClient(357): Not trying to connect to fc3fda29f660:38897 this server is in the failed servers list 2024-11-16T15:00:16,478 DEBUG [RPCClient-NioEventLoopGroup-6-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,eee,1731769026392.fd3944fc2b78246a853fb2232c747c35., hostname=fc3fda29f660,38897,1731769023879, seqNum=2 , the old value is region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,eee,1731769026392.fd3944fc2b78246a853fb2232c747c35., hostname=fc3fda29f660,38897,1731769023879, seqNum=2, error=org.apache.hadoop.hbase.ipc.FailedServerException: Call to address=fc3fda29f660:38897 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: fc3fda29f660:38897 2024-11-16T15:00:16,478 DEBUG [RPCClient-NioEventLoopGroup-6-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,eee,1731769026392.fd3944fc2b78246a853fb2232c747c35., hostname=fc3fda29f660,38897,1731769023879, seqNum=2 is org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: fc3fda29f660:38897 2024-11-16T15:00:16,478 DEBUG [RPCClient-NioEventLoopGroup-6-6 {}] client.AsyncRegionLocatorHelper(88): Try removing region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,eee,1731769026392.fd3944fc2b78246a853fb2232c747c35., hostname=fc3fda29f660,38897,1731769023879, seqNum=2 from cache 2024-11-16T15:00:16,592 DEBUG [RPCClient-NioEventLoopGroup-6-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='eee', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,eee,1731769026392.fd3944fc2b78246a853fb2232c747c35., hostname=fc3fda29f660,42985,1731769023951, seqNum=5] 2024-11-16T15:00:16,609 DEBUG [RPCClient-NioEventLoopGroup-6-5 {}] ipc.AbstractRpcClient(357): Not trying to connect to fc3fda29f660:38897 this server is in the failed servers list 2024-11-16T15:00:16,610 DEBUG [RPCClient-NioEventLoopGroup-6-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,iii,1731769026392.7352dcdbf9811dcebc51a714a1de4f55., hostname=fc3fda29f660,38897,1731769023879, seqNum=2 , the old value is region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,iii,1731769026392.7352dcdbf9811dcebc51a714a1de4f55., hostname=fc3fda29f660,38897,1731769023879, seqNum=2, error=org.apache.hadoop.hbase.ipc.FailedServerException: Call to address=fc3fda29f660:38897 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: fc3fda29f660:38897 2024-11-16T15:00:16,610 DEBUG [RPCClient-NioEventLoopGroup-6-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,iii,1731769026392.7352dcdbf9811dcebc51a714a1de4f55., hostname=fc3fda29f660,38897,1731769023879, seqNum=2 is org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: fc3fda29f660:38897 2024-11-16T15:00:16,610 DEBUG [RPCClient-NioEventLoopGroup-6-5 {}] client.AsyncRegionLocatorHelper(88): Try removing region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,iii,1731769026392.7352dcdbf9811dcebc51a714a1de4f55., hostname=fc3fda29f660,38897,1731769023879, seqNum=2 from cache 2024-11-16T15:00:16,725 DEBUG [RPCClient-NioEventLoopGroup-6-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='iii', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,iii,1731769026392.7352dcdbf9811dcebc51a714a1de4f55., hostname=fc3fda29f660,39089,1731769023769, seqNum=5] 2024-11-16T15:00:16,733 DEBUG [RPCClient-NioEventLoopGroup-6-6 {}] ipc.AbstractRpcClient(357): Not trying to connect to fc3fda29f660:38897 this server is in the failed servers list 2024-11-16T15:00:16,733 DEBUG [RPCClient-NioEventLoopGroup-6-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,jjj,1731769026392.7eb7b75d28b808c5c3f2f96e407b4347., hostname=fc3fda29f660,38897,1731769023879, seqNum=2 , the old value is region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,jjj,1731769026392.7eb7b75d28b808c5c3f2f96e407b4347., hostname=fc3fda29f660,38897,1731769023879, seqNum=2, error=org.apache.hadoop.hbase.ipc.FailedServerException: Call to address=fc3fda29f660:38897 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: fc3fda29f660:38897 2024-11-16T15:00:16,733 DEBUG [RPCClient-NioEventLoopGroup-6-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,jjj,1731769026392.7eb7b75d28b808c5c3f2f96e407b4347., hostname=fc3fda29f660,38897,1731769023879, seqNum=2 is org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: fc3fda29f660:38897 2024-11-16T15:00:16,733 DEBUG [RPCClient-NioEventLoopGroup-6-6 {}] client.AsyncRegionLocatorHelper(88): Try removing region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,jjj,1731769026392.7eb7b75d28b808c5c3f2f96e407b4347., hostname=fc3fda29f660,38897,1731769023879, seqNum=2 from cache 2024-11-16T15:00:16,844 DEBUG [RPCClient-NioEventLoopGroup-6-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='jjj', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,jjj,1731769026392.7eb7b75d28b808c5c3f2f96e407b4347., hostname=fc3fda29f660,42985,1731769023951, seqNum=5] 2024-11-16T15:00:16,860 DEBUG [RPCClient-NioEventLoopGroup-6-5 {}] ipc.AbstractRpcClient(357): Not trying to connect to fc3fda29f660:38897 this server is in the failed servers list 2024-11-16T15:00:16,860 DEBUG [RPCClient-NioEventLoopGroup-6-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,mmm,1731769026392.c2acbc05ae7bb1607a5719b701b40383., hostname=fc3fda29f660,38897,1731769023879, seqNum=2 , the old value is region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,mmm,1731769026392.c2acbc05ae7bb1607a5719b701b40383., hostname=fc3fda29f660,38897,1731769023879, seqNum=2, error=org.apache.hadoop.hbase.ipc.FailedServerException: Call to address=fc3fda29f660:38897 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: fc3fda29f660:38897 2024-11-16T15:00:16,860 DEBUG [RPCClient-NioEventLoopGroup-6-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,mmm,1731769026392.c2acbc05ae7bb1607a5719b701b40383., hostname=fc3fda29f660,38897,1731769023879, seqNum=2 is org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: fc3fda29f660:38897 2024-11-16T15:00:16,861 DEBUG [RPCClient-NioEventLoopGroup-6-5 {}] client.AsyncRegionLocatorHelper(88): Try removing region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,mmm,1731769026392.c2acbc05ae7bb1607a5719b701b40383., hostname=fc3fda29f660,38897,1731769023879, seqNum=2 from cache 2024-11-16T15:00:16,976 DEBUG [RPCClient-NioEventLoopGroup-6-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='mmm', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,mmm,1731769026392.c2acbc05ae7bb1607a5719b701b40383., hostname=fc3fda29f660,39089,1731769023769, seqNum=5] 2024-11-16T15:00:16,987 DEBUG [RPCClient-NioEventLoopGroup-6-5 {}] ipc.AbstractRpcClient(357): Not trying to connect to fc3fda29f660:38897 this server is in the failed servers list 2024-11-16T15:00:16,987 DEBUG [RPCClient-NioEventLoopGroup-6-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ppp,1731769026392.7f5c9df3a544933227f58b052f743f72., hostname=fc3fda29f660,38897,1731769023879, seqNum=2 , the old value is region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ppp,1731769026392.7f5c9df3a544933227f58b052f743f72., hostname=fc3fda29f660,38897,1731769023879, seqNum=2, error=org.apache.hadoop.hbase.ipc.FailedServerException: Call to address=fc3fda29f660:38897 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: fc3fda29f660:38897 2024-11-16T15:00:16,987 DEBUG [RPCClient-NioEventLoopGroup-6-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ppp,1731769026392.7f5c9df3a544933227f58b052f743f72., hostname=fc3fda29f660,38897,1731769023879, seqNum=2 is org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: fc3fda29f660:38897 2024-11-16T15:00:16,987 DEBUG [RPCClient-NioEventLoopGroup-6-5 {}] client.AsyncRegionLocatorHelper(88): Try removing region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ppp,1731769026392.7f5c9df3a544933227f58b052f743f72., hostname=fc3fda29f660,38897,1731769023879, seqNum=2 from cache 2024-11-16T15:00:17,096 DEBUG [RPCClient-NioEventLoopGroup-6-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='ppp', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ppp,1731769026392.7f5c9df3a544933227f58b052f743f72., hostname=fc3fda29f660,42985,1731769023951, seqNum=5] 2024-11-16T15:00:17,108 DEBUG [RPCClient-NioEventLoopGroup-6-5 {}] ipc.AbstractRpcClient(357): Not trying to connect to fc3fda29f660:38897 this server is in the failed servers list 2024-11-16T15:00:17,108 DEBUG [RPCClient-NioEventLoopGroup-6-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,sss,1731769026392.33054df03c1be51dc860405880be6075., hostname=fc3fda29f660,38897,1731769023879, seqNum=2 , the old value is region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,sss,1731769026392.33054df03c1be51dc860405880be6075., hostname=fc3fda29f660,38897,1731769023879, seqNum=2, error=org.apache.hadoop.hbase.ipc.FailedServerException: Call to address=fc3fda29f660:38897 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: fc3fda29f660:38897 2024-11-16T15:00:17,108 DEBUG [RPCClient-NioEventLoopGroup-6-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,sss,1731769026392.33054df03c1be51dc860405880be6075., hostname=fc3fda29f660,38897,1731769023879, seqNum=2 is org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: fc3fda29f660:38897 2024-11-16T15:00:17,108 DEBUG [RPCClient-NioEventLoopGroup-6-5 {}] client.AsyncRegionLocatorHelper(88): Try removing region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,sss,1731769026392.33054df03c1be51dc860405880be6075., hostname=fc3fda29f660,38897,1731769023879, seqNum=2 from cache 2024-11-16T15:00:17,226 DEBUG [RPCClient-NioEventLoopGroup-6-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='sss', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,sss,1731769026392.33054df03c1be51dc860405880be6075., hostname=fc3fda29f660,39089,1731769023769, seqNum=5] 2024-11-16T15:00:17,237 DEBUG [RPCClient-NioEventLoopGroup-6-5 {}] ipc.AbstractRpcClient(357): Not trying to connect to fc3fda29f660:38897 this server is in the failed servers list 2024-11-16T15:00:17,238 DEBUG [RPCClient-NioEventLoopGroup-6-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,vvv,1731769026392.b5566c7e10a0e001c7f58316bdd43c2f., hostname=fc3fda29f660,38897,1731769023879, seqNum=2 , the old value is region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,vvv,1731769026392.b5566c7e10a0e001c7f58316bdd43c2f., hostname=fc3fda29f660,38897,1731769023879, seqNum=2, error=org.apache.hadoop.hbase.ipc.FailedServerException: Call to address=fc3fda29f660:38897 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: fc3fda29f660:38897 2024-11-16T15:00:17,238 DEBUG [RPCClient-NioEventLoopGroup-6-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,vvv,1731769026392.b5566c7e10a0e001c7f58316bdd43c2f., hostname=fc3fda29f660,38897,1731769023879, seqNum=2 is org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: fc3fda29f660:38897 2024-11-16T15:00:17,238 DEBUG [RPCClient-NioEventLoopGroup-6-5 {}] client.AsyncRegionLocatorHelper(88): Try removing region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,vvv,1731769026392.b5566c7e10a0e001c7f58316bdd43c2f., hostname=fc3fda29f660,38897,1731769023879, seqNum=2 from cache 2024-11-16T15:00:17,349 DEBUG [RPCClient-NioEventLoopGroup-6-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='vvv', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,vvv,1731769026392.b5566c7e10a0e001c7f58316bdd43c2f., hostname=fc3fda29f660,42985,1731769023951, seqNum=5] 2024-11-16T15:00:17,360 DEBUG [RPCClient-NioEventLoopGroup-6-5 {}] ipc.AbstractRpcClient(357): Not trying to connect to fc3fda29f660:38897 this server is in the failed servers list 2024-11-16T15:00:17,360 DEBUG [RPCClient-NioEventLoopGroup-6-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,yyy,1731769026392.426ff231e487ce9f8492776ca1dacc85., hostname=fc3fda29f660,38897,1731769023879, seqNum=2 , the old value is region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,yyy,1731769026392.426ff231e487ce9f8492776ca1dacc85., hostname=fc3fda29f660,38897,1731769023879, seqNum=2, error=org.apache.hadoop.hbase.ipc.FailedServerException: Call to address=fc3fda29f660:38897 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: fc3fda29f660:38897 2024-11-16T15:00:17,360 DEBUG [RPCClient-NioEventLoopGroup-6-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,yyy,1731769026392.426ff231e487ce9f8492776ca1dacc85., hostname=fc3fda29f660,38897,1731769023879, seqNum=2 is org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: fc3fda29f660:38897 2024-11-16T15:00:17,360 DEBUG [RPCClient-NioEventLoopGroup-6-5 {}] client.AsyncRegionLocatorHelper(88): Try removing region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,yyy,1731769026392.426ff231e487ce9f8492776ca1dacc85., hostname=fc3fda29f660,38897,1731769023879, seqNum=2 from cache 2024-11-16T15:00:17,475 DEBUG [RPCClient-NioEventLoopGroup-6-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='yyy', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,yyy,1731769026392.426ff231e487ce9f8492776ca1dacc85., hostname=fc3fda29f660,39089,1731769023769, seqNum=5] 2024-11-16T15:00:17,546 WARN [Time-limited test {}] procedure2.ProcedureTestingUtility(193): Set Kill before store update to: false 2024-11-16T15:00:17,546 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster 2024-11-16T15:00:17,546 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-16T15:00:17,546 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79) at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611) at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.master.procedure.TestSCPBase.tearDown(TestSCPBase.java:72) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-16T15:00:17,547 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T15:00:17,547 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T15:00:17,547 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-16T15:00:17,547 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2024-11-16T15:00:17,547 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=450883051, stopped=false 2024-11-16T15:00:17,548 INFO [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=fc3fda29f660,43615,1731769023046 2024-11-16T15:00:17,603 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43615-0x101456fc9c70000, quorum=127.0.0.1:63224, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-16T15:00:17,603 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39089-0x101456fc9c70001, quorum=127.0.0.1:63224, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-16T15:00:17,603 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42985-0x101456fc9c70003, quorum=127.0.0.1:63224, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-16T15:00:17,603 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43615-0x101456fc9c70000, quorum=127.0.0.1:63224, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-16T15:00:17,603 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39089-0x101456fc9c70001, quorum=127.0.0.1:63224, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-16T15:00:17,604 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-16T15:00:17,604 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42985-0x101456fc9c70003, quorum=127.0.0.1:63224, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-16T15:00:17,604 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-16T15:00:17,605 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277) at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265) at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416) at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.master.procedure.TestSCPBase.tearDown(TestSCPBase.java:72) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-16T15:00:17,605 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T15:00:17,605 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:43615-0x101456fc9c70000, quorum=127.0.0.1:63224, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-16T15:00:17,605 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:39089-0x101456fc9c70001, quorum=127.0.0.1:63224, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-16T15:00:17,605 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:42985-0x101456fc9c70003, quorum=127.0.0.1:63224, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-16T15:00:17,606 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server 'fc3fda29f660,39089,1731769023769' ***** 2024-11-16T15:00:17,606 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-11-16T15:00:17,606 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server 'fc3fda29f660,42985,1731769023951' ***** 2024-11-16T15:00:17,606 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-11-16T15:00:17,606 INFO [RS:0;fc3fda29f660:39089 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-16T15:00:17,607 INFO [RS:2;fc3fda29f660:42985 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-16T15:00:17,607 INFO [RS:0;fc3fda29f660:39089 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-16T15:00:17,607 INFO [RS:0;fc3fda29f660:39089 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-16T15:00:17,607 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-16T15:00:17,607 INFO [RS:2;fc3fda29f660:42985 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-16T15:00:17,607 INFO [RS:2;fc3fda29f660:42985 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-16T15:00:17,607 INFO [RS:0;fc3fda29f660:39089 {}] regionserver.HRegionServer(3091): Received CLOSE for dde7f54607f11c5fb4271e361e82dabd 2024-11-16T15:00:17,607 INFO [RS:2;fc3fda29f660:42985 {}] regionserver.HRegionServer(3091): Received CLOSE for b5566c7e10a0e001c7f58316bdd43c2f 2024-11-16T15:00:17,608 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-16T15:00:17,608 INFO [RS:0;fc3fda29f660:39089 {}] regionserver.HRegionServer(3091): Received CLOSE for 7f11f79823720d664105d8643dfa862d 2024-11-16T15:00:17,608 INFO [RS:2;fc3fda29f660:42985 {}] regionserver.HRegionServer(3091): Received CLOSE for a5db9431e22152bd2a82e84504d2879a 2024-11-16T15:00:17,608 INFO [RS:0;fc3fda29f660:39089 {}] regionserver.HRegionServer(3091): Received CLOSE for 2f41a8c996224c818e917389bd3f5141 2024-11-16T15:00:17,608 INFO [RS:2;fc3fda29f660:42985 {}] regionserver.HRegionServer(3091): Received CLOSE for 35ef9a7f35bb0b08ec1d53b82d3f9777 2024-11-16T15:00:17,608 INFO [RS:0;fc3fda29f660:39089 {}] regionserver.HRegionServer(3091): Received CLOSE for 33054df03c1be51dc860405880be6075 2024-11-16T15:00:17,608 INFO [RS:2;fc3fda29f660:42985 {}] regionserver.HRegionServer(3091): Received CLOSE for fd3944fc2b78246a853fb2232c747c35 2024-11-16T15:00:17,608 INFO [RS:2;fc3fda29f660:42985 {}] regionserver.HRegionServer(3091): Received CLOSE for c3dde5f5f668d914608a2987d4f4c001 2024-11-16T15:00:17,608 INFO [RS:0;fc3fda29f660:39089 {}] regionserver.HRegionServer(3091): Received CLOSE for b4f73326a2a6a279d503e557c6ed452f 2024-11-16T15:00:17,608 INFO [RS:2;fc3fda29f660:42985 {}] regionserver.HRegionServer(3091): Received CLOSE for 7eb7b75d28b808c5c3f2f96e407b4347 2024-11-16T15:00:17,609 INFO [RS:0;fc3fda29f660:39089 {}] regionserver.HRegionServer(3091): Received CLOSE for 62a6d124ad9d161782c0632102c766de 2024-11-16T15:00:17,609 INFO [RS:2;fc3fda29f660:42985 {}] regionserver.HRegionServer(3091): Received CLOSE for dcd6414ddebad66e87ab37a6dba924f3 2024-11-16T15:00:17,609 INFO [RS:2;fc3fda29f660:42985 {}] regionserver.HRegionServer(3091): Received CLOSE for dc4fcaece350cd66f6bf88cfbdff6f15 2024-11-16T15:00:17,609 INFO [RS:0;fc3fda29f660:39089 {}] regionserver.HRegionServer(3091): Received CLOSE for 5bb22144956dcd43c9b1d957232b11e7 2024-11-16T15:00:17,609 INFO [RS:2;fc3fda29f660:42985 {}] regionserver.HRegionServer(3091): Received CLOSE for 7f5c9df3a544933227f58b052f743f72 2024-11-16T15:00:17,609 INFO [RS:2;fc3fda29f660:42985 {}] regionserver.HRegionServer(3091): Received CLOSE for 1d97df67e5110b0fe1a1e4b7c1341cbb 2024-11-16T15:00:17,609 INFO [RS:0;fc3fda29f660:39089 {}] regionserver.HRegionServer(3091): Received CLOSE for 18f03e2509a3a89e4a69ba87fad73940 2024-11-16T15:00:17,609 INFO [RS:2;fc3fda29f660:42985 {}] regionserver.HRegionServer(3091): Received CLOSE for 299131947f15f9c44bcfe22cd6623096 2024-11-16T15:00:17,609 INFO [RS:2;fc3fda29f660:42985 {}] regionserver.HRegionServer(3091): Received CLOSE for b364a006589fe0d2474ffa45692a6efc 2024-11-16T15:00:17,609 DEBUG [RS_CLOSE_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing dde7f54607f11c5fb4271e361e82dabd, disabling compactions & flushes 2024-11-16T15:00:17,609 INFO [RS:0;fc3fda29f660:39089 {}] regionserver.HRegionServer(3091): Received CLOSE for 4f0d635e77a1e3dd04c4d107bb0f1564 2024-11-16T15:00:17,609 DEBUG [RS_CLOSE_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing b5566c7e10a0e001c7f58316bdd43c2f, disabling compactions & flushes 2024-11-16T15:00:17,609 INFO [RS:2;fc3fda29f660:42985 {}] regionserver.HRegionServer(3091): Received CLOSE for 266565e60d7419a2a30ceec1b0f6519b 2024-11-16T15:00:17,609 INFO [RS_CLOSE_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ddd,1731769026392.dde7f54607f11c5fb4271e361e82dabd. 2024-11-16T15:00:17,609 INFO [RS:0;fc3fda29f660:39089 {}] regionserver.HRegionServer(3091): Received CLOSE for 994ca1526cf6c4838e65f85444853f1a 2024-11-16T15:00:17,609 INFO [RS_CLOSE_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,vvv,1731769026392.b5566c7e10a0e001c7f58316bdd43c2f. 2024-11-16T15:00:17,609 DEBUG [RS_CLOSE_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ddd,1731769026392.dde7f54607f11c5fb4271e361e82dabd. 2024-11-16T15:00:17,609 INFO [RS:2;fc3fda29f660:42985 {}] regionserver.HRegionServer(959): stopping server fc3fda29f660,42985,1731769023951 2024-11-16T15:00:17,609 INFO [RS:0;fc3fda29f660:39089 {}] regionserver.HRegionServer(3091): Received CLOSE for 426ff231e487ce9f8492776ca1dacc85 2024-11-16T15:00:17,609 DEBUG [RS_CLOSE_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,vvv,1731769026392.b5566c7e10a0e001c7f58316bdd43c2f. 2024-11-16T15:00:17,609 DEBUG [RS_CLOSE_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ddd,1731769026392.dde7f54607f11c5fb4271e361e82dabd. after waiting 0 ms 2024-11-16T15:00:17,609 INFO [RS:2;fc3fda29f660:42985 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-16T15:00:17,609 DEBUG [RS_CLOSE_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ddd,1731769026392.dde7f54607f11c5fb4271e361e82dabd. 2024-11-16T15:00:17,609 INFO [RS:0;fc3fda29f660:39089 {}] regionserver.HRegionServer(3091): Received CLOSE for 7352dcdbf9811dcebc51a714a1de4f55 2024-11-16T15:00:17,609 DEBUG [RS_CLOSE_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,vvv,1731769026392.b5566c7e10a0e001c7f58316bdd43c2f. after waiting 0 ms 2024-11-16T15:00:17,609 INFO [RS:2;fc3fda29f660:42985 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:2;fc3fda29f660:42985. 2024-11-16T15:00:17,609 DEBUG [RS_CLOSE_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,vvv,1731769026392.b5566c7e10a0e001c7f58316bdd43c2f. 2024-11-16T15:00:17,609 INFO [RS:0;fc3fda29f660:39089 {}] regionserver.HRegionServer(3091): Received CLOSE for c2acbc05ae7bb1607a5719b701b40383 2024-11-16T15:00:17,609 INFO [RS:0;fc3fda29f660:39089 {}] regionserver.HRegionServer(959): stopping server fc3fda29f660,39089,1731769023769 2024-11-16T15:00:17,609 DEBUG [RS:2;fc3fda29f660:42985 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-16T15:00:17,609 INFO [RS_CLOSE_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2902): Flushing dde7f54607f11c5fb4271e361e82dabd 3/3 column families, dataSize=32.95 KB heapSize=83.13 KB 2024-11-16T15:00:17,609 INFO [RS:0;fc3fda29f660:39089 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-16T15:00:17,609 DEBUG [RS:2;fc3fda29f660:42985 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T15:00:17,609 INFO [RS:0;fc3fda29f660:39089 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;fc3fda29f660:39089. 2024-11-16T15:00:17,609 DEBUG [RS:0;fc3fda29f660:39089 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-16T15:00:17,609 DEBUG [RS:0;fc3fda29f660:39089 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T15:00:17,609 INFO [RS:2;fc3fda29f660:42985 {}] regionserver.HRegionServer(1321): Waiting on 13 regions to close 2024-11-16T15:00:17,610 INFO [RS:0;fc3fda29f660:39089 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-16T15:00:17,610 INFO [RS:0;fc3fda29f660:39089 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-16T15:00:17,610 INFO [RS:0;fc3fda29f660:39089 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-16T15:00:17,610 INFO [RS:0;fc3fda29f660:39089 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-11-16T15:00:17,610 DEBUG [RS:2;fc3fda29f660:42985 {}] regionserver.HRegionServer(1351): Waiting on 1d97df67e5110b0fe1a1e4b7c1341cbb, 266565e60d7419a2a30ceec1b0f6519b, 299131947f15f9c44bcfe22cd6623096, 35ef9a7f35bb0b08ec1d53b82d3f9777, 7eb7b75d28b808c5c3f2f96e407b4347, 7f5c9df3a544933227f58b052f743f72, a5db9431e22152bd2a82e84504d2879a, b364a006589fe0d2474ffa45692a6efc, b5566c7e10a0e001c7f58316bdd43c2f, c3dde5f5f668d914608a2987d4f4c001, dc4fcaece350cd66f6bf88cfbdff6f15, dcd6414ddebad66e87ab37a6dba924f3, fd3944fc2b78246a853fb2232c747c35 2024-11-16T15:00:17,610 INFO [RS:0;fc3fda29f660:39089 {}] regionserver.HRegionServer(1321): Waiting on 14 regions to close 2024-11-16T15:00:17,611 DEBUG [RS:0;fc3fda29f660:39089 {}] regionserver.HRegionServer(1351): Waiting on 1588230740, 18f03e2509a3a89e4a69ba87fad73940, 2f41a8c996224c818e917389bd3f5141, 33054df03c1be51dc860405880be6075, 426ff231e487ce9f8492776ca1dacc85, 4f0d635e77a1e3dd04c4d107bb0f1564, 5bb22144956dcd43c9b1d957232b11e7, 62a6d124ad9d161782c0632102c766de, 7352dcdbf9811dcebc51a714a1de4f55, 7f11f79823720d664105d8643dfa862d, 994ca1526cf6c4838e65f85444853f1a, b4f73326a2a6a279d503e557c6ed452f, c2acbc05ae7bb1607a5719b701b40383, dde7f54607f11c5fb4271e361e82dabd 2024-11-16T15:00:17,611 DEBUG [RS_CLOSE_META-regionserver/fc3fda29f660:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-16T15:00:17,611 INFO [RS_CLOSE_META-regionserver/fc3fda29f660:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-16T15:00:17,611 DEBUG [RS_CLOSE_META-regionserver/fc3fda29f660:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-16T15:00:17,611 DEBUG [RS_CLOSE_META-regionserver/fc3fda29f660:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-16T15:00:17,611 DEBUG [RS_CLOSE_META-regionserver/fc3fda29f660:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-16T15:00:17,611 INFO [RS_CLOSE_META-regionserver/fc3fda29f660:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=13.37 KB heapSize=19.67 KB 2024-11-16T15:00:17,615 DEBUG [RS_CLOSE_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:42223/user/jenkins/test-data/a8176cb1-2ebb-b530-b3c7-53c9e53c186e/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/b5566c7e10a0e001c7f58316bdd43c2f/recovered.edits/7.seqid, newMaxSeqId=7, maxSeqId=4 2024-11-16T15:00:17,617 INFO [regionserver/fc3fda29f660:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-16T15:00:17,617 INFO [RS_CLOSE_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,vvv,1731769026392.b5566c7e10a0e001c7f58316bdd43c2f. 2024-11-16T15:00:17,617 INFO [regionserver/fc3fda29f660:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-16T15:00:17,617 DEBUG [RS_CLOSE_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for b5566c7e10a0e001c7f58316bdd43c2f: Waiting for close lock at 1731769217608Running coprocessor pre-close hooks at 1731769217609 (+1 ms)Disabling compacts and flushes for region at 1731769217609Disabling writes for close at 1731769217609Writing region close event to WAL at 1731769217611 (+2 ms)Running coprocessor post-close hooks at 1731769217615 (+4 ms)Closed at 1731769217617 (+2 ms) 2024-11-16T15:00:17,618 DEBUG [RS_CLOSE_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,vvv,1731769026392.b5566c7e10a0e001c7f58316bdd43c2f. 2024-11-16T15:00:17,618 DEBUG [RS_CLOSE_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing a5db9431e22152bd2a82e84504d2879a, disabling compactions & flushes 2024-11-16T15:00:17,618 INFO [RS_CLOSE_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,fff,1731769026392.a5db9431e22152bd2a82e84504d2879a. 2024-11-16T15:00:17,618 DEBUG [RS_CLOSE_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,fff,1731769026392.a5db9431e22152bd2a82e84504d2879a. 2024-11-16T15:00:17,618 DEBUG [RS_CLOSE_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,fff,1731769026392.a5db9431e22152bd2a82e84504d2879a. after waiting 0 ms 2024-11-16T15:00:17,618 DEBUG [RS_CLOSE_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,fff,1731769026392.a5db9431e22152bd2a82e84504d2879a. 2024-11-16T15:00:17,618 INFO [RS_CLOSE_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2902): Flushing a5db9431e22152bd2a82e84504d2879a 3/3 column families, dataSize=32.95 KB heapSize=83.13 KB 2024-11-16T15:00:17,623 DEBUG [RS_CLOSE_META-regionserver/fc3fda29f660:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42223/user/jenkins/test-data/a8176cb1-2ebb-b530-b3c7-53c9e53c186e/data/hbase/meta/1588230740/.tmp/info/dfa5a465bfba4509965d219b268e1cfa is 268, key is testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,eee,1731769026392.fd3944fc2b78246a853fb2232c747c35./info:regioninfo/1731769215610/Put/seqid=0 2024-11-16T15:00:17,634 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36531 is added to blk_1073741887_1063 (size=19812) 2024-11-16T15:00:17,634 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41671 is added to blk_1073741887_1063 (size=19812) 2024-11-16T15:00:17,634 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40641 is added to blk_1073741887_1063 (size=19812) 2024-11-16T15:00:17,634 DEBUG [RS_CLOSE_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42223/user/jenkins/test-data/a8176cb1-2ebb-b530-b3c7-53c9e53c186e/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/dde7f54607f11c5fb4271e361e82dabd/.tmp/colfamily11/46848e841af54ac69cb4ca616994565d is 52, key is ddd/colfamily11:colfamily11/1731769030078/Put/seqid=0 2024-11-16T15:00:17,635 INFO [RS_CLOSE_META-regionserver/fc3fda29f660:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.37 KB at sequenceid=81 (bloomFilter=true), to=hdfs://localhost:42223/user/jenkins/test-data/a8176cb1-2ebb-b530-b3c7-53c9e53c186e/data/hbase/meta/1588230740/.tmp/info/dfa5a465bfba4509965d219b268e1cfa 2024-11-16T15:00:17,637 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-16T15:00:17,640 DEBUG [RS_CLOSE_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42223/user/jenkins/test-data/a8176cb1-2ebb-b530-b3c7-53c9e53c186e/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/a5db9431e22152bd2a82e84504d2879a/.tmp/colfamily11/e8e8cba1758348bdbeb8d844933dd75a is 52, key is fff/colfamily11:colfamily11/1731769030080/Put/seqid=0 2024-11-16T15:00:17,645 DEBUG [RS_CLOSE_META-regionserver/fc3fda29f660:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42223/user/jenkins/test-data/a8176cb1-2ebb-b530-b3c7-53c9e53c186e/data/hbase/meta/1588230740/.tmp/info/dfa5a465bfba4509965d219b268e1cfa as hdfs://localhost:42223/user/jenkins/test-data/a8176cb1-2ebb-b530-b3c7-53c9e53c186e/data/hbase/meta/1588230740/info/dfa5a465bfba4509965d219b268e1cfa 2024-11-16T15:00:17,652 INFO [RS_CLOSE_META-regionserver/fc3fda29f660:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42223/user/jenkins/test-data/a8176cb1-2ebb-b530-b3c7-53c9e53c186e/data/hbase/meta/1588230740/info/dfa5a465bfba4509965d219b268e1cfa, entries=72, sequenceid=81, filesize=19.3 K 2024-11-16T15:00:17,654 INFO [RS_CLOSE_META-regionserver/fc3fda29f660:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~13.37 KB/13686, heapSize ~18.91 KB/19360, currentSize=0 B/0 for 1588230740 in 43ms, sequenceid=81, compaction requested=false 2024-11-16T15:00:17,655 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40641 is added to blk_1073741888_1064 (size=39263) 2024-11-16T15:00:17,655 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41671 is added to blk_1073741888_1064 (size=39263) 2024-11-16T15:00:17,656 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36531 is added to blk_1073741888_1064 (size=39263) 2024-11-16T15:00:17,658 INFO [RS_CLOSE_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=32.95 KB at sequenceid=5 (bloomFilter=false), to=hdfs://localhost:42223/user/jenkins/test-data/a8176cb1-2ebb-b530-b3c7-53c9e53c186e/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/dde7f54607f11c5fb4271e361e82dabd/.tmp/colfamily11/46848e841af54ac69cb4ca616994565d 2024-11-16T15:00:17,666 DEBUG [RS_CLOSE_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42223/user/jenkins/test-data/a8176cb1-2ebb-b530-b3c7-53c9e53c186e/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/dde7f54607f11c5fb4271e361e82dabd/.tmp/colfamily11/46848e841af54ac69cb4ca616994565d as hdfs://localhost:42223/user/jenkins/test-data/a8176cb1-2ebb-b530-b3c7-53c9e53c186e/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/dde7f54607f11c5fb4271e361e82dabd/colfamily11/46848e841af54ac69cb4ca616994565d 2024-11-16T15:00:17,673 INFO [RS_CLOSE_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42223/user/jenkins/test-data/a8176cb1-2ebb-b530-b3c7-53c9e53c186e/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/dde7f54607f11c5fb4271e361e82dabd/colfamily11/46848e841af54ac69cb4ca616994565d, entries=703, sequenceid=5, filesize=38.3 K 2024-11-16T15:00:17,676 INFO [RS_CLOSE_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3140): Finished flush of dataSize ~32.95 KB/33744, heapSize ~82.62 KB/84600, currentSize=0 B/0 for dde7f54607f11c5fb4271e361e82dabd in 67ms, sequenceid=5, compaction requested=false 2024-11-16T15:00:17,680 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36531 is added to blk_1073741889_1065 (size=39263) 2024-11-16T15:00:17,680 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40641 is added to blk_1073741889_1065 (size=39263) 2024-11-16T15:00:17,680 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41671 is added to blk_1073741889_1065 (size=39263) 2024-11-16T15:00:17,683 DEBUG [RS_CLOSE_META-regionserver/fc3fda29f660:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:42223/user/jenkins/test-data/a8176cb1-2ebb-b530-b3c7-53c9e53c186e/data/hbase/meta/1588230740/recovered.edits/84.seqid, newMaxSeqId=84, maxSeqId=60 2024-11-16T15:00:17,684 DEBUG [RS_CLOSE_META-regionserver/fc3fda29f660:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-16T15:00:17,684 INFO [RS_CLOSE_META-regionserver/fc3fda29f660:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-16T15:00:17,684 DEBUG [RS_CLOSE_META-regionserver/fc3fda29f660:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1731769217611Running coprocessor pre-close hooks at 1731769217611Disabling compacts and flushes for region at 1731769217611Disabling writes for close at 1731769217611Obtaining lock to block concurrent updates at 1731769217611Preparing flush snapshotting stores in 1588230740 at 1731769217611Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=13686, getHeapSize=20080, getOffHeapSize=0, getCellsCount=72 at 1731769217615 (+4 ms)Flushing stores of hbase:meta,,1.1588230740 at 1731769217618 (+3 ms)Flushing 1588230740/info: creating writer at 1731769217618Flushing 1588230740/info: appending metadata at 1731769217622 (+4 ms)Flushing 1588230740/info: closing flushed file at 1731769217623 (+1 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@1e6b120b: reopening flushed file at 1731769217644 (+21 ms)Finished flush of dataSize ~13.37 KB/13686, heapSize ~18.91 KB/19360, currentSize=0 B/0 for 1588230740 in 43ms, sequenceid=81, compaction requested=false at 1731769217654 (+10 ms)Writing region close event to WAL at 1731769217679 (+25 ms)Running coprocessor post-close hooks at 1731769217684 (+5 ms)Closed at 1731769217684 2024-11-16T15:00:17,685 DEBUG [RS_CLOSE_META-regionserver/fc3fda29f660:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-11-16T15:00:17,687 DEBUG [RS_CLOSE_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:42223/user/jenkins/test-data/a8176cb1-2ebb-b530-b3c7-53c9e53c186e/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/dde7f54607f11c5fb4271e361e82dabd/recovered.edits/8.seqid, newMaxSeqId=8, maxSeqId=1 2024-11-16T15:00:17,688 INFO [RS_CLOSE_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ddd,1731769026392.dde7f54607f11c5fb4271e361e82dabd. 2024-11-16T15:00:17,688 DEBUG [RS_CLOSE_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for dde7f54607f11c5fb4271e361e82dabd: Waiting for close lock at 1731769217608Running coprocessor pre-close hooks at 1731769217609 (+1 ms)Disabling compacts and flushes for region at 1731769217609Disabling writes for close at 1731769217609Obtaining lock to block concurrent updates at 1731769217609Preparing flush snapshotting stores in dde7f54607f11c5fb4271e361e82dabd at 1731769217609Finished memstore snapshotting testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ddd,1731769026392.dde7f54607f11c5fb4271e361e82dabd., syncing WAL and waiting on mvcc, flushsize=dataSize=33744, getHeapSize=85080, getOffHeapSize=0, getCellsCount=703 at 1731769217615 (+6 ms)Flushing stores of testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ddd,1731769026392.dde7f54607f11c5fb4271e361e82dabd. at 1731769217618 (+3 ms)Flushing dde7f54607f11c5fb4271e361e82dabd/colfamily11: creating writer at 1731769217618Flushing dde7f54607f11c5fb4271e361e82dabd/colfamily11: appending metadata at 1731769217634 (+16 ms)Flushing dde7f54607f11c5fb4271e361e82dabd/colfamily11: closing flushed file at 1731769217634Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@4eadef72: reopening flushed file at 1731769217665 (+31 ms)Finished flush of dataSize ~32.95 KB/33744, heapSize ~82.62 KB/84600, currentSize=0 B/0 for dde7f54607f11c5fb4271e361e82dabd in 67ms, sequenceid=5, compaction requested=false at 1731769217676 (+11 ms)Writing region close event to WAL at 1731769217681 (+5 ms)Running coprocessor post-close hooks at 1731769217688 (+7 ms)Closed at 1731769217688 2024-11-16T15:00:17,688 DEBUG [RS_CLOSE_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ddd,1731769026392.dde7f54607f11c5fb4271e361e82dabd. 2024-11-16T15:00:17,688 DEBUG [RS_CLOSE_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 7f11f79823720d664105d8643dfa862d, disabling compactions & flushes 2024-11-16T15:00:17,688 INFO [RS_CLOSE_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ttt,1731769026392.7f11f79823720d664105d8643dfa862d. 2024-11-16T15:00:17,688 DEBUG [RS_CLOSE_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ttt,1731769026392.7f11f79823720d664105d8643dfa862d. 2024-11-16T15:00:17,688 DEBUG [RS_CLOSE_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ttt,1731769026392.7f11f79823720d664105d8643dfa862d. after waiting 0 ms 2024-11-16T15:00:17,688 DEBUG [RS_CLOSE_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ttt,1731769026392.7f11f79823720d664105d8643dfa862d. 2024-11-16T15:00:17,689 INFO [RS_CLOSE_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2902): Flushing 7f11f79823720d664105d8643dfa862d 3/3 column families, dataSize=32.95 KB heapSize=83.13 KB 2024-11-16T15:00:17,704 DEBUG [RS_CLOSE_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42223/user/jenkins/test-data/a8176cb1-2ebb-b530-b3c7-53c9e53c186e/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/7f11f79823720d664105d8643dfa862d/.tmp/colfamily11/eb902045b4ae40dfaf1357cc4590a66d is 52, key is ttt/colfamily11:colfamily11/1731769030322/Put/seqid=0 2024-11-16T15:00:17,711 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41671 is added to blk_1073741890_1066 (size=39263) 2024-11-16T15:00:17,711 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40641 is added to blk_1073741890_1066 (size=39263) 2024-11-16T15:00:17,711 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36531 is added to blk_1073741890_1066 (size=39263) 2024-11-16T15:00:17,712 INFO [RS_CLOSE_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=32.95 KB at sequenceid=5 (bloomFilter=false), to=hdfs://localhost:42223/user/jenkins/test-data/a8176cb1-2ebb-b530-b3c7-53c9e53c186e/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/7f11f79823720d664105d8643dfa862d/.tmp/colfamily11/eb902045b4ae40dfaf1357cc4590a66d 2024-11-16T15:00:17,718 DEBUG [RS_CLOSE_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42223/user/jenkins/test-data/a8176cb1-2ebb-b530-b3c7-53c9e53c186e/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/7f11f79823720d664105d8643dfa862d/.tmp/colfamily11/eb902045b4ae40dfaf1357cc4590a66d as hdfs://localhost:42223/user/jenkins/test-data/a8176cb1-2ebb-b530-b3c7-53c9e53c186e/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/7f11f79823720d664105d8643dfa862d/colfamily11/eb902045b4ae40dfaf1357cc4590a66d 2024-11-16T15:00:17,724 INFO [RS_CLOSE_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42223/user/jenkins/test-data/a8176cb1-2ebb-b530-b3c7-53c9e53c186e/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/7f11f79823720d664105d8643dfa862d/colfamily11/eb902045b4ae40dfaf1357cc4590a66d, entries=703, sequenceid=5, filesize=38.3 K 2024-11-16T15:00:17,725 INFO [RS_CLOSE_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3140): Finished flush of dataSize ~32.95 KB/33744, heapSize ~82.62 KB/84600, currentSize=0 B/0 for 7f11f79823720d664105d8643dfa862d in 37ms, sequenceid=5, compaction requested=false 2024-11-16T15:00:17,730 DEBUG [RS_CLOSE_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:42223/user/jenkins/test-data/a8176cb1-2ebb-b530-b3c7-53c9e53c186e/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/7f11f79823720d664105d8643dfa862d/recovered.edits/8.seqid, newMaxSeqId=8, maxSeqId=1 2024-11-16T15:00:17,730 INFO [RS_CLOSE_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ttt,1731769026392.7f11f79823720d664105d8643dfa862d. 2024-11-16T15:00:17,730 DEBUG [RS_CLOSE_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 7f11f79823720d664105d8643dfa862d: Waiting for close lock at 1731769217688Running coprocessor pre-close hooks at 1731769217688Disabling compacts and flushes for region at 1731769217688Disabling writes for close at 1731769217688Obtaining lock to block concurrent updates at 1731769217689 (+1 ms)Preparing flush snapshotting stores in 7f11f79823720d664105d8643dfa862d at 1731769217689Finished memstore snapshotting testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ttt,1731769026392.7f11f79823720d664105d8643dfa862d., syncing WAL and waiting on mvcc, flushsize=dataSize=33744, getHeapSize=85080, getOffHeapSize=0, getCellsCount=703 at 1731769217689Flushing stores of testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ttt,1731769026392.7f11f79823720d664105d8643dfa862d. at 1731769217689Flushing 7f11f79823720d664105d8643dfa862d/colfamily11: creating writer at 1731769217689Flushing 7f11f79823720d664105d8643dfa862d/colfamily11: appending metadata at 1731769217704 (+15 ms)Flushing 7f11f79823720d664105d8643dfa862d/colfamily11: closing flushed file at 1731769217704Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@1d5f0f63: reopening flushed file at 1731769217717 (+13 ms)Finished flush of dataSize ~32.95 KB/33744, heapSize ~82.62 KB/84600, currentSize=0 B/0 for 7f11f79823720d664105d8643dfa862d in 37ms, sequenceid=5, compaction requested=false at 1731769217725 (+8 ms)Writing region close event to WAL at 1731769217726 (+1 ms)Running coprocessor post-close hooks at 1731769217730 (+4 ms)Closed at 1731769217730 2024-11-16T15:00:17,730 DEBUG [RS_CLOSE_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ttt,1731769026392.7f11f79823720d664105d8643dfa862d. 2024-11-16T15:00:17,731 DEBUG [RS_CLOSE_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 2f41a8c996224c818e917389bd3f5141, disabling compactions & flushes 2024-11-16T15:00:17,731 INFO [RS_CLOSE_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,zzz,1731769026392.2f41a8c996224c818e917389bd3f5141. 2024-11-16T15:00:17,731 DEBUG [RS_CLOSE_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,zzz,1731769026392.2f41a8c996224c818e917389bd3f5141. 2024-11-16T15:00:17,731 DEBUG [RS_CLOSE_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,zzz,1731769026392.2f41a8c996224c818e917389bd3f5141. after waiting 0 ms 2024-11-16T15:00:17,731 DEBUG [RS_CLOSE_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,zzz,1731769026392.2f41a8c996224c818e917389bd3f5141. 2024-11-16T15:00:17,731 INFO [RS_CLOSE_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2902): Flushing 2f41a8c996224c818e917389bd3f5141 3/3 column families, dataSize=48 B heapSize=888 B 2024-11-16T15:00:17,745 DEBUG [RS_CLOSE_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42223/user/jenkins/test-data/a8176cb1-2ebb-b530-b3c7-53c9e53c186e/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/2f41a8c996224c818e917389bd3f5141/.tmp/colfamily11/a831a05e13914a579aac1141bfdbae5a is 52, key is zzz/colfamily11:colfamily11/1731769030368/Put/seqid=0 2024-11-16T15:00:17,751 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36531 is added to blk_1073741891_1067 (size=4857) 2024-11-16T15:00:17,751 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41671 is added to blk_1073741891_1067 (size=4857) 2024-11-16T15:00:17,751 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40641 is added to blk_1073741891_1067 (size=4857) 2024-11-16T15:00:17,752 INFO [RS_CLOSE_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=48 B at sequenceid=5 (bloomFilter=false), to=hdfs://localhost:42223/user/jenkins/test-data/a8176cb1-2ebb-b530-b3c7-53c9e53c186e/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/2f41a8c996224c818e917389bd3f5141/.tmp/colfamily11/a831a05e13914a579aac1141bfdbae5a 2024-11-16T15:00:17,758 DEBUG [RS_CLOSE_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42223/user/jenkins/test-data/a8176cb1-2ebb-b530-b3c7-53c9e53c186e/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/2f41a8c996224c818e917389bd3f5141/.tmp/colfamily11/a831a05e13914a579aac1141bfdbae5a as hdfs://localhost:42223/user/jenkins/test-data/a8176cb1-2ebb-b530-b3c7-53c9e53c186e/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/2f41a8c996224c818e917389bd3f5141/colfamily11/a831a05e13914a579aac1141bfdbae5a 2024-11-16T15:00:17,765 INFO [RS_CLOSE_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42223/user/jenkins/test-data/a8176cb1-2ebb-b530-b3c7-53c9e53c186e/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/2f41a8c996224c818e917389bd3f5141/colfamily11/a831a05e13914a579aac1141bfdbae5a, entries=1, sequenceid=5, filesize=4.7 K 2024-11-16T15:00:17,766 INFO [RS_CLOSE_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3140): Finished flush of dataSize ~48 B/48, heapSize ~360 B/360, currentSize=0 B/0 for 2f41a8c996224c818e917389bd3f5141 in 35ms, sequenceid=5, compaction requested=false 2024-11-16T15:00:17,770 DEBUG [RS_CLOSE_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:42223/user/jenkins/test-data/a8176cb1-2ebb-b530-b3c7-53c9e53c186e/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/2f41a8c996224c818e917389bd3f5141/recovered.edits/8.seqid, newMaxSeqId=8, maxSeqId=1 2024-11-16T15:00:17,771 INFO [RS_CLOSE_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,zzz,1731769026392.2f41a8c996224c818e917389bd3f5141. 2024-11-16T15:00:17,771 DEBUG [RS_CLOSE_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 2f41a8c996224c818e917389bd3f5141: Waiting for close lock at 1731769217730Running coprocessor pre-close hooks at 1731769217730Disabling compacts and flushes for region at 1731769217730Disabling writes for close at 1731769217731 (+1 ms)Obtaining lock to block concurrent updates at 1731769217731Preparing flush snapshotting stores in 2f41a8c996224c818e917389bd3f5141 at 1731769217731Finished memstore snapshotting testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,zzz,1731769026392.2f41a8c996224c818e917389bd3f5141., syncing WAL and waiting on mvcc, flushsize=dataSize=48, getHeapSize=840, getOffHeapSize=0, getCellsCount=1 at 1731769217731Flushing stores of testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,zzz,1731769026392.2f41a8c996224c818e917389bd3f5141. at 1731769217732 (+1 ms)Flushing 2f41a8c996224c818e917389bd3f5141/colfamily11: creating writer at 1731769217732Flushing 2f41a8c996224c818e917389bd3f5141/colfamily11: appending metadata at 1731769217745 (+13 ms)Flushing 2f41a8c996224c818e917389bd3f5141/colfamily11: closing flushed file at 1731769217745Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@60e099c8: reopening flushed file at 1731769217758 (+13 ms)Finished flush of dataSize ~48 B/48, heapSize ~360 B/360, currentSize=0 B/0 for 2f41a8c996224c818e917389bd3f5141 in 35ms, sequenceid=5, compaction requested=false at 1731769217766 (+8 ms)Writing region close event to WAL at 1731769217767 (+1 ms)Running coprocessor post-close hooks at 1731769217771 (+4 ms)Closed at 1731769217771 2024-11-16T15:00:17,771 DEBUG [RS_CLOSE_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,zzz,1731769026392.2f41a8c996224c818e917389bd3f5141. 2024-11-16T15:00:17,771 DEBUG [RS_CLOSE_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 33054df03c1be51dc860405880be6075, disabling compactions & flushes 2024-11-16T15:00:17,771 INFO [RS_CLOSE_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,sss,1731769026392.33054df03c1be51dc860405880be6075. 2024-11-16T15:00:17,772 DEBUG [RS_CLOSE_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,sss,1731769026392.33054df03c1be51dc860405880be6075. 2024-11-16T15:00:17,772 DEBUG [RS_CLOSE_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,sss,1731769026392.33054df03c1be51dc860405880be6075. after waiting 0 ms 2024-11-16T15:00:17,772 DEBUG [RS_CLOSE_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,sss,1731769026392.33054df03c1be51dc860405880be6075. 2024-11-16T15:00:17,776 DEBUG [RS_CLOSE_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:42223/user/jenkins/test-data/a8176cb1-2ebb-b530-b3c7-53c9e53c186e/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/33054df03c1be51dc860405880be6075/recovered.edits/7.seqid, newMaxSeqId=7, maxSeqId=4 2024-11-16T15:00:17,776 INFO [RS_CLOSE_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,sss,1731769026392.33054df03c1be51dc860405880be6075. 2024-11-16T15:00:17,777 DEBUG [RS_CLOSE_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 33054df03c1be51dc860405880be6075: Waiting for close lock at 1731769217771Running coprocessor pre-close hooks at 1731769217771Disabling compacts and flushes for region at 1731769217771Disabling writes for close at 1731769217772 (+1 ms)Writing region close event to WAL at 1731769217772Running coprocessor post-close hooks at 1731769217776 (+4 ms)Closed at 1731769217776 2024-11-16T15:00:17,777 DEBUG [RS_CLOSE_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,sss,1731769026392.33054df03c1be51dc860405880be6075. 2024-11-16T15:00:17,777 DEBUG [RS_CLOSE_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing b4f73326a2a6a279d503e557c6ed452f, disabling compactions & flushes 2024-11-16T15:00:17,777 INFO [RS_CLOSE_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,kkk,1731769026392.b4f73326a2a6a279d503e557c6ed452f. 2024-11-16T15:00:17,777 DEBUG [RS_CLOSE_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,kkk,1731769026392.b4f73326a2a6a279d503e557c6ed452f. 2024-11-16T15:00:17,777 DEBUG [RS_CLOSE_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,kkk,1731769026392.b4f73326a2a6a279d503e557c6ed452f. after waiting 0 ms 2024-11-16T15:00:17,777 DEBUG [RS_CLOSE_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,kkk,1731769026392.b4f73326a2a6a279d503e557c6ed452f. 2024-11-16T15:00:17,777 INFO [RS_CLOSE_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2902): Flushing b4f73326a2a6a279d503e557c6ed452f 3/3 column families, dataSize=32.95 KB heapSize=83.13 KB 2024-11-16T15:00:17,793 DEBUG [RS_CLOSE_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42223/user/jenkins/test-data/a8176cb1-2ebb-b530-b3c7-53c9e53c186e/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/b4f73326a2a6a279d503e557c6ed452f/.tmp/colfamily11/b69320334cc64ceb91f02139e40735cf is 52, key is kkk/colfamily11:colfamily11/1731769030147/Put/seqid=0 2024-11-16T15:00:17,799 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41671 is added to blk_1073741892_1068 (size=39263) 2024-11-16T15:00:17,799 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36531 is added to blk_1073741892_1068 (size=39263) 2024-11-16T15:00:17,800 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40641 is added to blk_1073741892_1068 (size=39263) 2024-11-16T15:00:17,800 INFO [RS_CLOSE_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=32.95 KB at sequenceid=5 (bloomFilter=false), to=hdfs://localhost:42223/user/jenkins/test-data/a8176cb1-2ebb-b530-b3c7-53c9e53c186e/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/b4f73326a2a6a279d503e557c6ed452f/.tmp/colfamily11/b69320334cc64ceb91f02139e40735cf 2024-11-16T15:00:17,806 DEBUG [RS_CLOSE_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42223/user/jenkins/test-data/a8176cb1-2ebb-b530-b3c7-53c9e53c186e/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/b4f73326a2a6a279d503e557c6ed452f/.tmp/colfamily11/b69320334cc64ceb91f02139e40735cf as hdfs://localhost:42223/user/jenkins/test-data/a8176cb1-2ebb-b530-b3c7-53c9e53c186e/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/b4f73326a2a6a279d503e557c6ed452f/colfamily11/b69320334cc64ceb91f02139e40735cf 2024-11-16T15:00:17,810 DEBUG [RS:2;fc3fda29f660:42985 {}] regionserver.HRegionServer(1351): Waiting on 1d97df67e5110b0fe1a1e4b7c1341cbb, 266565e60d7419a2a30ceec1b0f6519b, 299131947f15f9c44bcfe22cd6623096, 35ef9a7f35bb0b08ec1d53b82d3f9777, 7eb7b75d28b808c5c3f2f96e407b4347, 7f5c9df3a544933227f58b052f743f72, a5db9431e22152bd2a82e84504d2879a, b364a006589fe0d2474ffa45692a6efc, c3dde5f5f668d914608a2987d4f4c001, dc4fcaece350cd66f6bf88cfbdff6f15, dcd6414ddebad66e87ab37a6dba924f3, fd3944fc2b78246a853fb2232c747c35 2024-11-16T15:00:17,811 DEBUG [RS:0;fc3fda29f660:39089 {}] regionserver.HRegionServer(1351): Waiting on 18f03e2509a3a89e4a69ba87fad73940, 426ff231e487ce9f8492776ca1dacc85, 4f0d635e77a1e3dd04c4d107bb0f1564, 5bb22144956dcd43c9b1d957232b11e7, 62a6d124ad9d161782c0632102c766de, 7352dcdbf9811dcebc51a714a1de4f55, 994ca1526cf6c4838e65f85444853f1a, b4f73326a2a6a279d503e557c6ed452f, c2acbc05ae7bb1607a5719b701b40383 2024-11-16T15:00:17,813 INFO [RS_CLOSE_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42223/user/jenkins/test-data/a8176cb1-2ebb-b530-b3c7-53c9e53c186e/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/b4f73326a2a6a279d503e557c6ed452f/colfamily11/b69320334cc64ceb91f02139e40735cf, entries=703, sequenceid=5, filesize=38.3 K 2024-11-16T15:00:17,814 INFO [RS_CLOSE_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3140): Finished flush of dataSize ~32.95 KB/33744, heapSize ~82.62 KB/84600, currentSize=0 B/0 for b4f73326a2a6a279d503e557c6ed452f in 36ms, sequenceid=5, compaction requested=false 2024-11-16T15:00:17,817 DEBUG [RS_CLOSE_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:42223/user/jenkins/test-data/a8176cb1-2ebb-b530-b3c7-53c9e53c186e/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/b4f73326a2a6a279d503e557c6ed452f/recovered.edits/8.seqid, newMaxSeqId=8, maxSeqId=1 2024-11-16T15:00:17,818 INFO [RS_CLOSE_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,kkk,1731769026392.b4f73326a2a6a279d503e557c6ed452f. 2024-11-16T15:00:17,818 DEBUG [RS_CLOSE_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for b4f73326a2a6a279d503e557c6ed452f: Waiting for close lock at 1731769217777Running coprocessor pre-close hooks at 1731769217777Disabling compacts and flushes for region at 1731769217777Disabling writes for close at 1731769217777Obtaining lock to block concurrent updates at 1731769217777Preparing flush snapshotting stores in b4f73326a2a6a279d503e557c6ed452f at 1731769217777Finished memstore snapshotting testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,kkk,1731769026392.b4f73326a2a6a279d503e557c6ed452f., syncing WAL and waiting on mvcc, flushsize=dataSize=33744, getHeapSize=85080, getOffHeapSize=0, getCellsCount=703 at 1731769217777Flushing stores of testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,kkk,1731769026392.b4f73326a2a6a279d503e557c6ed452f. at 1731769217778 (+1 ms)Flushing b4f73326a2a6a279d503e557c6ed452f/colfamily11: creating writer at 1731769217778Flushing b4f73326a2a6a279d503e557c6ed452f/colfamily11: appending metadata at 1731769217792 (+14 ms)Flushing b4f73326a2a6a279d503e557c6ed452f/colfamily11: closing flushed file at 1731769217792Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@63a56885: reopening flushed file at 1731769217805 (+13 ms)Finished flush of dataSize ~32.95 KB/33744, heapSize ~82.62 KB/84600, currentSize=0 B/0 for b4f73326a2a6a279d503e557c6ed452f in 36ms, sequenceid=5, compaction requested=false at 1731769217814 (+9 ms)Writing region close event to WAL at 1731769217814Running coprocessor post-close hooks at 1731769217818 (+4 ms)Closed at 1731769217818 2024-11-16T15:00:17,818 DEBUG [RS_CLOSE_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,kkk,1731769026392.b4f73326a2a6a279d503e557c6ed452f. 2024-11-16T15:00:17,819 DEBUG [RS_CLOSE_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 62a6d124ad9d161782c0632102c766de, disabling compactions & flushes 2024-11-16T15:00:17,819 INFO [RS_CLOSE_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,www,1731769026392.62a6d124ad9d161782c0632102c766de. 2024-11-16T15:00:17,819 DEBUG [RS_CLOSE_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,www,1731769026392.62a6d124ad9d161782c0632102c766de. 2024-11-16T15:00:17,819 DEBUG [RS_CLOSE_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,www,1731769026392.62a6d124ad9d161782c0632102c766de. after waiting 0 ms 2024-11-16T15:00:17,819 DEBUG [RS_CLOSE_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,www,1731769026392.62a6d124ad9d161782c0632102c766de. 2024-11-16T15:00:17,819 INFO [RS_CLOSE_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2902): Flushing 62a6d124ad9d161782c0632102c766de 3/3 column families, dataSize=32.95 KB heapSize=83.13 KB 2024-11-16T15:00:17,835 DEBUG [RS_CLOSE_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42223/user/jenkins/test-data/a8176cb1-2ebb-b530-b3c7-53c9e53c186e/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/62a6d124ad9d161782c0632102c766de/.tmp/colfamily11/320e533904444f95b6de1c6981f8b1a7 is 52, key is www/colfamily11:colfamily11/1731769030360/Put/seqid=0 2024-11-16T15:00:17,841 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36531 is added to blk_1073741893_1069 (size=39263) 2024-11-16T15:00:17,842 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41671 is added to blk_1073741893_1069 (size=39263) 2024-11-16T15:00:17,842 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40641 is added to blk_1073741893_1069 (size=39263) 2024-11-16T15:00:17,842 INFO [RS_CLOSE_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=32.95 KB at sequenceid=5 (bloomFilter=false), to=hdfs://localhost:42223/user/jenkins/test-data/a8176cb1-2ebb-b530-b3c7-53c9e53c186e/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/62a6d124ad9d161782c0632102c766de/.tmp/colfamily11/320e533904444f95b6de1c6981f8b1a7 2024-11-16T15:00:17,849 DEBUG [RS_CLOSE_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42223/user/jenkins/test-data/a8176cb1-2ebb-b530-b3c7-53c9e53c186e/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/62a6d124ad9d161782c0632102c766de/.tmp/colfamily11/320e533904444f95b6de1c6981f8b1a7 as hdfs://localhost:42223/user/jenkins/test-data/a8176cb1-2ebb-b530-b3c7-53c9e53c186e/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/62a6d124ad9d161782c0632102c766de/colfamily11/320e533904444f95b6de1c6981f8b1a7 2024-11-16T15:00:17,857 INFO [RS_CLOSE_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42223/user/jenkins/test-data/a8176cb1-2ebb-b530-b3c7-53c9e53c186e/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/62a6d124ad9d161782c0632102c766de/colfamily11/320e533904444f95b6de1c6981f8b1a7, entries=703, sequenceid=5, filesize=38.3 K 2024-11-16T15:00:17,858 INFO [RS_CLOSE_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3140): Finished flush of dataSize ~32.95 KB/33744, heapSize ~82.62 KB/84600, currentSize=0 B/0 for 62a6d124ad9d161782c0632102c766de in 39ms, sequenceid=5, compaction requested=false 2024-11-16T15:00:17,863 DEBUG [RS_CLOSE_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:42223/user/jenkins/test-data/a8176cb1-2ebb-b530-b3c7-53c9e53c186e/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/62a6d124ad9d161782c0632102c766de/recovered.edits/8.seqid, newMaxSeqId=8, maxSeqId=1 2024-11-16T15:00:17,863 INFO [RS_CLOSE_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,www,1731769026392.62a6d124ad9d161782c0632102c766de. 2024-11-16T15:00:17,863 DEBUG [RS_CLOSE_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 62a6d124ad9d161782c0632102c766de: Waiting for close lock at 1731769217819Running coprocessor pre-close hooks at 1731769217819Disabling compacts and flushes for region at 1731769217819Disabling writes for close at 1731769217819Obtaining lock to block concurrent updates at 1731769217819Preparing flush snapshotting stores in 62a6d124ad9d161782c0632102c766de at 1731769217819Finished memstore snapshotting testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,www,1731769026392.62a6d124ad9d161782c0632102c766de., syncing WAL and waiting on mvcc, flushsize=dataSize=33744, getHeapSize=85080, getOffHeapSize=0, getCellsCount=703 at 1731769217819Flushing stores of testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,www,1731769026392.62a6d124ad9d161782c0632102c766de. at 1731769217820 (+1 ms)Flushing 62a6d124ad9d161782c0632102c766de/colfamily11: creating writer at 1731769217820Flushing 62a6d124ad9d161782c0632102c766de/colfamily11: appending metadata at 1731769217835 (+15 ms)Flushing 62a6d124ad9d161782c0632102c766de/colfamily11: closing flushed file at 1731769217835Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@7d6c99f1: reopening flushed file at 1731769217848 (+13 ms)Finished flush of dataSize ~32.95 KB/33744, heapSize ~82.62 KB/84600, currentSize=0 B/0 for 62a6d124ad9d161782c0632102c766de in 39ms, sequenceid=5, compaction requested=false at 1731769217858 (+10 ms)Writing region close event to WAL at 1731769217859 (+1 ms)Running coprocessor post-close hooks at 1731769217863 (+4 ms)Closed at 1731769217863 2024-11-16T15:00:17,863 DEBUG [RS_CLOSE_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,www,1731769026392.62a6d124ad9d161782c0632102c766de. 2024-11-16T15:00:17,864 DEBUG [RS_CLOSE_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 5bb22144956dcd43c9b1d957232b11e7, disabling compactions & flushes 2024-11-16T15:00:17,864 INFO [RS_CLOSE_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ccc,1731769026392.5bb22144956dcd43c9b1d957232b11e7. 2024-11-16T15:00:17,864 DEBUG [RS_CLOSE_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ccc,1731769026392.5bb22144956dcd43c9b1d957232b11e7. 2024-11-16T15:00:17,864 DEBUG [RS_CLOSE_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ccc,1731769026392.5bb22144956dcd43c9b1d957232b11e7. after waiting 0 ms 2024-11-16T15:00:17,864 DEBUG [RS_CLOSE_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ccc,1731769026392.5bb22144956dcd43c9b1d957232b11e7. 2024-11-16T15:00:17,864 INFO [RS_CLOSE_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2902): Flushing 5bb22144956dcd43c9b1d957232b11e7 3/3 column families, dataSize=32.95 KB heapSize=83.13 KB 2024-11-16T15:00:17,880 DEBUG [RS_CLOSE_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42223/user/jenkins/test-data/a8176cb1-2ebb-b530-b3c7-53c9e53c186e/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/5bb22144956dcd43c9b1d957232b11e7/.tmp/colfamily11/ac37d76f28684cd3b0dbb19d133ecd7e is 52, key is ccc/colfamily11:colfamily11/1731769030021/Put/seqid=0 2024-11-16T15:00:17,886 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41671 is added to blk_1073741894_1070 (size=39263) 2024-11-16T15:00:17,886 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36531 is added to blk_1073741894_1070 (size=39263) 2024-11-16T15:00:17,887 INFO [RS_CLOSE_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=32.95 KB at sequenceid=5 (bloomFilter=false), to=hdfs://localhost:42223/user/jenkins/test-data/a8176cb1-2ebb-b530-b3c7-53c9e53c186e/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/5bb22144956dcd43c9b1d957232b11e7/.tmp/colfamily11/ac37d76f28684cd3b0dbb19d133ecd7e 2024-11-16T15:00:17,888 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40641 is added to blk_1073741894_1070 (size=39263) 2024-11-16T15:00:17,893 DEBUG [RS_CLOSE_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42223/user/jenkins/test-data/a8176cb1-2ebb-b530-b3c7-53c9e53c186e/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/5bb22144956dcd43c9b1d957232b11e7/.tmp/colfamily11/ac37d76f28684cd3b0dbb19d133ecd7e as hdfs://localhost:42223/user/jenkins/test-data/a8176cb1-2ebb-b530-b3c7-53c9e53c186e/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/5bb22144956dcd43c9b1d957232b11e7/colfamily11/ac37d76f28684cd3b0dbb19d133ecd7e 2024-11-16T15:00:17,898 INFO [RS_CLOSE_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42223/user/jenkins/test-data/a8176cb1-2ebb-b530-b3c7-53c9e53c186e/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/5bb22144956dcd43c9b1d957232b11e7/colfamily11/ac37d76f28684cd3b0dbb19d133ecd7e, entries=703, sequenceid=5, filesize=38.3 K 2024-11-16T15:00:17,899 INFO [RS_CLOSE_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3140): Finished flush of dataSize ~32.95 KB/33744, heapSize ~82.62 KB/84600, currentSize=0 B/0 for 5bb22144956dcd43c9b1d957232b11e7 in 35ms, sequenceid=5, compaction requested=false 2024-11-16T15:00:17,903 DEBUG [RS_CLOSE_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:42223/user/jenkins/test-data/a8176cb1-2ebb-b530-b3c7-53c9e53c186e/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/5bb22144956dcd43c9b1d957232b11e7/recovered.edits/8.seqid, newMaxSeqId=8, maxSeqId=1 2024-11-16T15:00:17,904 INFO [RS_CLOSE_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ccc,1731769026392.5bb22144956dcd43c9b1d957232b11e7. 2024-11-16T15:00:17,904 DEBUG [RS_CLOSE_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 5bb22144956dcd43c9b1d957232b11e7: Waiting for close lock at 1731769217864Running coprocessor pre-close hooks at 1731769217864Disabling compacts and flushes for region at 1731769217864Disabling writes for close at 1731769217864Obtaining lock to block concurrent updates at 1731769217864Preparing flush snapshotting stores in 5bb22144956dcd43c9b1d957232b11e7 at 1731769217864Finished memstore snapshotting testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ccc,1731769026392.5bb22144956dcd43c9b1d957232b11e7., syncing WAL and waiting on mvcc, flushsize=dataSize=33744, getHeapSize=85080, getOffHeapSize=0, getCellsCount=703 at 1731769217864Flushing stores of testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ccc,1731769026392.5bb22144956dcd43c9b1d957232b11e7. at 1731769217865 (+1 ms)Flushing 5bb22144956dcd43c9b1d957232b11e7/colfamily11: creating writer at 1731769217865Flushing 5bb22144956dcd43c9b1d957232b11e7/colfamily11: appending metadata at 1731769217880 (+15 ms)Flushing 5bb22144956dcd43c9b1d957232b11e7/colfamily11: closing flushed file at 1731769217880Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@15fa308e: reopening flushed file at 1731769217892 (+12 ms)Finished flush of dataSize ~32.95 KB/33744, heapSize ~82.62 KB/84600, currentSize=0 B/0 for 5bb22144956dcd43c9b1d957232b11e7 in 35ms, sequenceid=5, compaction requested=false at 1731769217899 (+7 ms)Writing region close event to WAL at 1731769217900 (+1 ms)Running coprocessor post-close hooks at 1731769217904 (+4 ms)Closed at 1731769217904 2024-11-16T15:00:17,904 DEBUG [RS_CLOSE_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ccc,1731769026392.5bb22144956dcd43c9b1d957232b11e7. 2024-11-16T15:00:17,904 DEBUG [RS_CLOSE_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 18f03e2509a3a89e4a69ba87fad73940, disabling compactions & flushes 2024-11-16T15:00:17,904 INFO [RS_CLOSE_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,qqq,1731769026392.18f03e2509a3a89e4a69ba87fad73940. 2024-11-16T15:00:17,904 DEBUG [RS_CLOSE_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,qqq,1731769026392.18f03e2509a3a89e4a69ba87fad73940. 2024-11-16T15:00:17,905 DEBUG [RS_CLOSE_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,qqq,1731769026392.18f03e2509a3a89e4a69ba87fad73940. after waiting 0 ms 2024-11-16T15:00:17,905 DEBUG [RS_CLOSE_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,qqq,1731769026392.18f03e2509a3a89e4a69ba87fad73940. 2024-11-16T15:00:17,905 INFO [RS_CLOSE_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2902): Flushing 18f03e2509a3a89e4a69ba87fad73940 3/3 column families, dataSize=32.95 KB heapSize=83.13 KB 2024-11-16T15:00:17,922 DEBUG [RS_CLOSE_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42223/user/jenkins/test-data/a8176cb1-2ebb-b530-b3c7-53c9e53c186e/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/18f03e2509a3a89e4a69ba87fad73940/.tmp/colfamily11/cd20ad927262488c96114409ea09e274 is 52, key is qqq/colfamily11:colfamily11/1731769030280/Put/seqid=0 2024-11-16T15:00:17,928 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40641 is added to blk_1073741895_1071 (size=39263) 2024-11-16T15:00:17,928 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36531 is added to blk_1073741895_1071 (size=39263) 2024-11-16T15:00:17,931 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41671 is added to blk_1073741895_1071 (size=39263) 2024-11-16T15:00:17,931 INFO [RS_CLOSE_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=32.95 KB at sequenceid=5 (bloomFilter=false), to=hdfs://localhost:42223/user/jenkins/test-data/a8176cb1-2ebb-b530-b3c7-53c9e53c186e/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/18f03e2509a3a89e4a69ba87fad73940/.tmp/colfamily11/cd20ad927262488c96114409ea09e274 2024-11-16T15:00:17,937 DEBUG [RS_CLOSE_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42223/user/jenkins/test-data/a8176cb1-2ebb-b530-b3c7-53c9e53c186e/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/18f03e2509a3a89e4a69ba87fad73940/.tmp/colfamily11/cd20ad927262488c96114409ea09e274 as hdfs://localhost:42223/user/jenkins/test-data/a8176cb1-2ebb-b530-b3c7-53c9e53c186e/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/18f03e2509a3a89e4a69ba87fad73940/colfamily11/cd20ad927262488c96114409ea09e274 2024-11-16T15:00:17,943 INFO [RS_CLOSE_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42223/user/jenkins/test-data/a8176cb1-2ebb-b530-b3c7-53c9e53c186e/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/18f03e2509a3a89e4a69ba87fad73940/colfamily11/cd20ad927262488c96114409ea09e274, entries=703, sequenceid=5, filesize=38.3 K 2024-11-16T15:00:17,944 INFO [RS_CLOSE_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3140): Finished flush of dataSize ~32.95 KB/33744, heapSize ~82.62 KB/84600, currentSize=0 B/0 for 18f03e2509a3a89e4a69ba87fad73940 in 39ms, sequenceid=5, compaction requested=false 2024-11-16T15:00:17,949 DEBUG [RS_CLOSE_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:42223/user/jenkins/test-data/a8176cb1-2ebb-b530-b3c7-53c9e53c186e/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/18f03e2509a3a89e4a69ba87fad73940/recovered.edits/8.seqid, newMaxSeqId=8, maxSeqId=1 2024-11-16T15:00:17,950 INFO [RS_CLOSE_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,qqq,1731769026392.18f03e2509a3a89e4a69ba87fad73940. 2024-11-16T15:00:17,950 DEBUG [RS_CLOSE_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 18f03e2509a3a89e4a69ba87fad73940: Waiting for close lock at 1731769217904Running coprocessor pre-close hooks at 1731769217904Disabling compacts and flushes for region at 1731769217904Disabling writes for close at 1731769217905 (+1 ms)Obtaining lock to block concurrent updates at 1731769217905Preparing flush snapshotting stores in 18f03e2509a3a89e4a69ba87fad73940 at 1731769217905Finished memstore snapshotting testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,qqq,1731769026392.18f03e2509a3a89e4a69ba87fad73940., syncing WAL and waiting on mvcc, flushsize=dataSize=33744, getHeapSize=85080, getOffHeapSize=0, getCellsCount=703 at 1731769217905Flushing stores of testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,qqq,1731769026392.18f03e2509a3a89e4a69ba87fad73940. at 1731769217906 (+1 ms)Flushing 18f03e2509a3a89e4a69ba87fad73940/colfamily11: creating writer at 1731769217906Flushing 18f03e2509a3a89e4a69ba87fad73940/colfamily11: appending metadata at 1731769217922 (+16 ms)Flushing 18f03e2509a3a89e4a69ba87fad73940/colfamily11: closing flushed file at 1731769217922Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@63e022bc: reopening flushed file at 1731769217936 (+14 ms)Finished flush of dataSize ~32.95 KB/33744, heapSize ~82.62 KB/84600, currentSize=0 B/0 for 18f03e2509a3a89e4a69ba87fad73940 in 39ms, sequenceid=5, compaction requested=false at 1731769217944 (+8 ms)Writing region close event to WAL at 1731769217945 (+1 ms)Running coprocessor post-close hooks at 1731769217950 (+5 ms)Closed at 1731769217950 2024-11-16T15:00:17,950 DEBUG [RS_CLOSE_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,qqq,1731769026392.18f03e2509a3a89e4a69ba87fad73940. 2024-11-16T15:00:17,951 DEBUG [RS_CLOSE_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 4f0d635e77a1e3dd04c4d107bb0f1564, disabling compactions & flushes 2024-11-16T15:00:17,951 INFO [RS_CLOSE_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,,1731769026392.4f0d635e77a1e3dd04c4d107bb0f1564. 2024-11-16T15:00:17,951 DEBUG [RS_CLOSE_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,,1731769026392.4f0d635e77a1e3dd04c4d107bb0f1564. 2024-11-16T15:00:17,951 DEBUG [RS_CLOSE_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,,1731769026392.4f0d635e77a1e3dd04c4d107bb0f1564. after waiting 0 ms 2024-11-16T15:00:17,951 DEBUG [RS_CLOSE_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,,1731769026392.4f0d635e77a1e3dd04c4d107bb0f1564. 2024-11-16T15:00:17,956 DEBUG [RS_CLOSE_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:42223/user/jenkins/test-data/a8176cb1-2ebb-b530-b3c7-53c9e53c186e/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/4f0d635e77a1e3dd04c4d107bb0f1564/recovered.edits/7.seqid, newMaxSeqId=7, maxSeqId=4 2024-11-16T15:00:17,957 INFO [RS_CLOSE_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,,1731769026392.4f0d635e77a1e3dd04c4d107bb0f1564. 2024-11-16T15:00:17,957 DEBUG [RS_CLOSE_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 4f0d635e77a1e3dd04c4d107bb0f1564: Waiting for close lock at 1731769217951Running coprocessor pre-close hooks at 1731769217951Disabling compacts and flushes for region at 1731769217951Disabling writes for close at 1731769217951Writing region close event to WAL at 1731769217952 (+1 ms)Running coprocessor post-close hooks at 1731769217957 (+5 ms)Closed at 1731769217957 2024-11-16T15:00:17,957 DEBUG [RS_CLOSE_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,,1731769026392.4f0d635e77a1e3dd04c4d107bb0f1564. 2024-11-16T15:00:17,957 DEBUG [RS_CLOSE_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 994ca1526cf6c4838e65f85444853f1a, disabling compactions & flushes 2024-11-16T15:00:17,957 INFO [RS_CLOSE_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,nnn,1731769026392.994ca1526cf6c4838e65f85444853f1a. 2024-11-16T15:00:17,957 DEBUG [RS_CLOSE_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,nnn,1731769026392.994ca1526cf6c4838e65f85444853f1a. 2024-11-16T15:00:17,957 DEBUG [RS_CLOSE_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,nnn,1731769026392.994ca1526cf6c4838e65f85444853f1a. after waiting 0 ms 2024-11-16T15:00:17,957 DEBUG [RS_CLOSE_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,nnn,1731769026392.994ca1526cf6c4838e65f85444853f1a. 2024-11-16T15:00:17,957 INFO [RS_CLOSE_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2902): Flushing 994ca1526cf6c4838e65f85444853f1a 3/3 column families, dataSize=32.95 KB heapSize=83.13 KB 2024-11-16T15:00:17,978 DEBUG [RS_CLOSE_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42223/user/jenkins/test-data/a8176cb1-2ebb-b530-b3c7-53c9e53c186e/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/994ca1526cf6c4838e65f85444853f1a/.tmp/colfamily11/822b7a5481534b508182702febeb31c9 is 52, key is nnn/colfamily11:colfamily11/1731769030205/Put/seqid=0 2024-11-16T15:00:17,983 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41671 is added to blk_1073741896_1072 (size=39263) 2024-11-16T15:00:17,984 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40641 is added to blk_1073741896_1072 (size=39263) 2024-11-16T15:00:17,984 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36531 is added to blk_1073741896_1072 (size=39263) 2024-11-16T15:00:17,984 INFO [RS_CLOSE_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=32.95 KB at sequenceid=5 (bloomFilter=false), to=hdfs://localhost:42223/user/jenkins/test-data/a8176cb1-2ebb-b530-b3c7-53c9e53c186e/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/994ca1526cf6c4838e65f85444853f1a/.tmp/colfamily11/822b7a5481534b508182702febeb31c9 2024-11-16T15:00:17,990 DEBUG [RS_CLOSE_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42223/user/jenkins/test-data/a8176cb1-2ebb-b530-b3c7-53c9e53c186e/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/994ca1526cf6c4838e65f85444853f1a/.tmp/colfamily11/822b7a5481534b508182702febeb31c9 as hdfs://localhost:42223/user/jenkins/test-data/a8176cb1-2ebb-b530-b3c7-53c9e53c186e/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/994ca1526cf6c4838e65f85444853f1a/colfamily11/822b7a5481534b508182702febeb31c9 2024-11-16T15:00:17,996 INFO [RS_CLOSE_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42223/user/jenkins/test-data/a8176cb1-2ebb-b530-b3c7-53c9e53c186e/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/994ca1526cf6c4838e65f85444853f1a/colfamily11/822b7a5481534b508182702febeb31c9, entries=703, sequenceid=5, filesize=38.3 K 2024-11-16T15:00:17,997 INFO [RS_CLOSE_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3140): Finished flush of dataSize ~32.95 KB/33744, heapSize ~82.62 KB/84600, currentSize=0 B/0 for 994ca1526cf6c4838e65f85444853f1a in 39ms, sequenceid=5, compaction requested=false 2024-11-16T15:00:18,000 DEBUG [RS_CLOSE_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:42223/user/jenkins/test-data/a8176cb1-2ebb-b530-b3c7-53c9e53c186e/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/994ca1526cf6c4838e65f85444853f1a/recovered.edits/8.seqid, newMaxSeqId=8, maxSeqId=1 2024-11-16T15:00:18,001 INFO [RS_CLOSE_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,nnn,1731769026392.994ca1526cf6c4838e65f85444853f1a. 2024-11-16T15:00:18,001 DEBUG [RS_CLOSE_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 994ca1526cf6c4838e65f85444853f1a: Waiting for close lock at 1731769217957Running coprocessor pre-close hooks at 1731769217957Disabling compacts and flushes for region at 1731769217957Disabling writes for close at 1731769217957Obtaining lock to block concurrent updates at 1731769217957Preparing flush snapshotting stores in 994ca1526cf6c4838e65f85444853f1a at 1731769217957Finished memstore snapshotting testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,nnn,1731769026392.994ca1526cf6c4838e65f85444853f1a., syncing WAL and waiting on mvcc, flushsize=dataSize=33744, getHeapSize=85080, getOffHeapSize=0, getCellsCount=703 at 1731769217958 (+1 ms)Flushing stores of testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,nnn,1731769026392.994ca1526cf6c4838e65f85444853f1a. at 1731769217958Flushing 994ca1526cf6c4838e65f85444853f1a/colfamily11: creating writer at 1731769217958Flushing 994ca1526cf6c4838e65f85444853f1a/colfamily11: appending metadata at 1731769217977 (+19 ms)Flushing 994ca1526cf6c4838e65f85444853f1a/colfamily11: closing flushed file at 1731769217977Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@813418a: reopening flushed file at 1731769217989 (+12 ms)Finished flush of dataSize ~32.95 KB/33744, heapSize ~82.62 KB/84600, currentSize=0 B/0 for 994ca1526cf6c4838e65f85444853f1a in 39ms, sequenceid=5, compaction requested=false at 1731769217997 (+8 ms)Writing region close event to WAL at 1731769217997Running coprocessor post-close hooks at 1731769218001 (+4 ms)Closed at 1731769218001 2024-11-16T15:00:18,001 DEBUG [RS_CLOSE_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,nnn,1731769026392.994ca1526cf6c4838e65f85444853f1a. 2024-11-16T15:00:18,001 DEBUG [RS_CLOSE_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 426ff231e487ce9f8492776ca1dacc85, disabling compactions & flushes 2024-11-16T15:00:18,001 INFO [RS_CLOSE_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,yyy,1731769026392.426ff231e487ce9f8492776ca1dacc85. 2024-11-16T15:00:18,001 DEBUG [RS_CLOSE_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,yyy,1731769026392.426ff231e487ce9f8492776ca1dacc85. 2024-11-16T15:00:18,001 DEBUG [RS_CLOSE_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,yyy,1731769026392.426ff231e487ce9f8492776ca1dacc85. after waiting 0 ms 2024-11-16T15:00:18,002 DEBUG [RS_CLOSE_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,yyy,1731769026392.426ff231e487ce9f8492776ca1dacc85. 2024-11-16T15:00:18,006 DEBUG [RS_CLOSE_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:42223/user/jenkins/test-data/a8176cb1-2ebb-b530-b3c7-53c9e53c186e/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/426ff231e487ce9f8492776ca1dacc85/recovered.edits/7.seqid, newMaxSeqId=7, maxSeqId=4 2024-11-16T15:00:18,006 INFO [RS_CLOSE_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,yyy,1731769026392.426ff231e487ce9f8492776ca1dacc85. 2024-11-16T15:00:18,006 DEBUG [RS_CLOSE_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 426ff231e487ce9f8492776ca1dacc85: Waiting for close lock at 1731769218001Running coprocessor pre-close hooks at 1731769218001Disabling compacts and flushes for region at 1731769218001Disabling writes for close at 1731769218002 (+1 ms)Writing region close event to WAL at 1731769218002Running coprocessor post-close hooks at 1731769218006 (+4 ms)Closed at 1731769218006 2024-11-16T15:00:18,006 DEBUG [RS_CLOSE_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,yyy,1731769026392.426ff231e487ce9f8492776ca1dacc85. 2024-11-16T15:00:18,007 DEBUG [RS_CLOSE_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 7352dcdbf9811dcebc51a714a1de4f55, disabling compactions & flushes 2024-11-16T15:00:18,007 INFO [RS_CLOSE_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,iii,1731769026392.7352dcdbf9811dcebc51a714a1de4f55. 2024-11-16T15:00:18,007 DEBUG [RS_CLOSE_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,iii,1731769026392.7352dcdbf9811dcebc51a714a1de4f55. 2024-11-16T15:00:18,007 DEBUG [RS_CLOSE_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,iii,1731769026392.7352dcdbf9811dcebc51a714a1de4f55. after waiting 0 ms 2024-11-16T15:00:18,007 DEBUG [RS_CLOSE_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,iii,1731769026392.7352dcdbf9811dcebc51a714a1de4f55. 2024-11-16T15:00:18,010 DEBUG [RS:2;fc3fda29f660:42985 {}] regionserver.HRegionServer(1351): Waiting on 1d97df67e5110b0fe1a1e4b7c1341cbb, 266565e60d7419a2a30ceec1b0f6519b, 299131947f15f9c44bcfe22cd6623096, 35ef9a7f35bb0b08ec1d53b82d3f9777, 7eb7b75d28b808c5c3f2f96e407b4347, 7f5c9df3a544933227f58b052f743f72, a5db9431e22152bd2a82e84504d2879a, b364a006589fe0d2474ffa45692a6efc, c3dde5f5f668d914608a2987d4f4c001, dc4fcaece350cd66f6bf88cfbdff6f15, dcd6414ddebad66e87ab37a6dba924f3, fd3944fc2b78246a853fb2232c747c35 2024-11-16T15:00:18,011 DEBUG [RS_CLOSE_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:42223/user/jenkins/test-data/a8176cb1-2ebb-b530-b3c7-53c9e53c186e/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/7352dcdbf9811dcebc51a714a1de4f55/recovered.edits/7.seqid, newMaxSeqId=7, maxSeqId=4 2024-11-16T15:00:18,011 DEBUG [RS:0;fc3fda29f660:39089 {}] regionserver.HRegionServer(1351): Waiting on 7352dcdbf9811dcebc51a714a1de4f55, c2acbc05ae7bb1607a5719b701b40383 2024-11-16T15:00:18,011 INFO [RS_CLOSE_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,iii,1731769026392.7352dcdbf9811dcebc51a714a1de4f55. 2024-11-16T15:00:18,011 DEBUG [RS_CLOSE_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 7352dcdbf9811dcebc51a714a1de4f55: Waiting for close lock at 1731769218007Running coprocessor pre-close hooks at 1731769218007Disabling compacts and flushes for region at 1731769218007Disabling writes for close at 1731769218007Writing region close event to WAL at 1731769218007Running coprocessor post-close hooks at 1731769218011 (+4 ms)Closed at 1731769218011 2024-11-16T15:00:18,011 DEBUG [RS_CLOSE_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,iii,1731769026392.7352dcdbf9811dcebc51a714a1de4f55. 2024-11-16T15:00:18,012 DEBUG [RS_CLOSE_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing c2acbc05ae7bb1607a5719b701b40383, disabling compactions & flushes 2024-11-16T15:00:18,012 INFO [RS_CLOSE_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,mmm,1731769026392.c2acbc05ae7bb1607a5719b701b40383. 2024-11-16T15:00:18,012 DEBUG [RS_CLOSE_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,mmm,1731769026392.c2acbc05ae7bb1607a5719b701b40383. 2024-11-16T15:00:18,012 DEBUG [RS_CLOSE_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,mmm,1731769026392.c2acbc05ae7bb1607a5719b701b40383. after waiting 0 ms 2024-11-16T15:00:18,012 DEBUG [RS_CLOSE_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,mmm,1731769026392.c2acbc05ae7bb1607a5719b701b40383. 2024-11-16T15:00:18,016 DEBUG [RS_CLOSE_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:42223/user/jenkins/test-data/a8176cb1-2ebb-b530-b3c7-53c9e53c186e/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/c2acbc05ae7bb1607a5719b701b40383/recovered.edits/7.seqid, newMaxSeqId=7, maxSeqId=4 2024-11-16T15:00:18,017 INFO [RS_CLOSE_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,mmm,1731769026392.c2acbc05ae7bb1607a5719b701b40383. 2024-11-16T15:00:18,017 DEBUG [RS_CLOSE_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for c2acbc05ae7bb1607a5719b701b40383: Waiting for close lock at 1731769218012Running coprocessor pre-close hooks at 1731769218012Disabling compacts and flushes for region at 1731769218012Disabling writes for close at 1731769218012Writing region close event to WAL at 1731769218013 (+1 ms)Running coprocessor post-close hooks at 1731769218017 (+4 ms)Closed at 1731769218017 2024-11-16T15:00:18,017 DEBUG [RS_CLOSE_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,mmm,1731769026392.c2acbc05ae7bb1607a5719b701b40383. 2024-11-16T15:00:18,081 INFO [RS_CLOSE_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=32.95 KB at sequenceid=5 (bloomFilter=false), to=hdfs://localhost:42223/user/jenkins/test-data/a8176cb1-2ebb-b530-b3c7-53c9e53c186e/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/a5db9431e22152bd2a82e84504d2879a/.tmp/colfamily11/e8e8cba1758348bdbeb8d844933dd75a 2024-11-16T15:00:18,087 DEBUG [RS_CLOSE_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42223/user/jenkins/test-data/a8176cb1-2ebb-b530-b3c7-53c9e53c186e/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/a5db9431e22152bd2a82e84504d2879a/.tmp/colfamily11/e8e8cba1758348bdbeb8d844933dd75a as hdfs://localhost:42223/user/jenkins/test-data/a8176cb1-2ebb-b530-b3c7-53c9e53c186e/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/a5db9431e22152bd2a82e84504d2879a/colfamily11/e8e8cba1758348bdbeb8d844933dd75a 2024-11-16T15:00:18,093 INFO [RS_CLOSE_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42223/user/jenkins/test-data/a8176cb1-2ebb-b530-b3c7-53c9e53c186e/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/a5db9431e22152bd2a82e84504d2879a/colfamily11/e8e8cba1758348bdbeb8d844933dd75a, entries=703, sequenceid=5, filesize=38.3 K 2024-11-16T15:00:18,094 INFO [RS_CLOSE_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3140): Finished flush of dataSize ~32.95 KB/33744, heapSize ~82.62 KB/84600, currentSize=0 B/0 for a5db9431e22152bd2a82e84504d2879a in 475ms, sequenceid=5, compaction requested=false 2024-11-16T15:00:18,098 DEBUG [RS_CLOSE_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:42223/user/jenkins/test-data/a8176cb1-2ebb-b530-b3c7-53c9e53c186e/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/a5db9431e22152bd2a82e84504d2879a/recovered.edits/8.seqid, newMaxSeqId=8, maxSeqId=1 2024-11-16T15:00:18,098 INFO [RS_CLOSE_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,fff,1731769026392.a5db9431e22152bd2a82e84504d2879a. 2024-11-16T15:00:18,098 DEBUG [RS_CLOSE_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for a5db9431e22152bd2a82e84504d2879a: Waiting for close lock at 1731769217618Running coprocessor pre-close hooks at 1731769217618Disabling compacts and flushes for region at 1731769217618Disabling writes for close at 1731769217618Obtaining lock to block concurrent updates at 1731769217618Preparing flush snapshotting stores in a5db9431e22152bd2a82e84504d2879a at 1731769217618Finished memstore snapshotting testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,fff,1731769026392.a5db9431e22152bd2a82e84504d2879a., syncing WAL and waiting on mvcc, flushsize=dataSize=33744, getHeapSize=85080, getOffHeapSize=0, getCellsCount=703 at 1731769217619 (+1 ms)Flushing stores of testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,fff,1731769026392.a5db9431e22152bd2a82e84504d2879a. at 1731769217619Flushing a5db9431e22152bd2a82e84504d2879a/colfamily11: creating writer at 1731769217619Flushing a5db9431e22152bd2a82e84504d2879a/colfamily11: appending metadata at 1731769217640 (+21 ms)Flushing a5db9431e22152bd2a82e84504d2879a/colfamily11: closing flushed file at 1731769217640Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@1972f5aa: reopening flushed file at 1731769218086 (+446 ms)Finished flush of dataSize ~32.95 KB/33744, heapSize ~82.62 KB/84600, currentSize=0 B/0 for a5db9431e22152bd2a82e84504d2879a in 475ms, sequenceid=5, compaction requested=false at 1731769218094 (+8 ms)Writing region close event to WAL at 1731769218094Running coprocessor post-close hooks at 1731769218098 (+4 ms)Closed at 1731769218098 2024-11-16T15:00:18,099 DEBUG [RS_CLOSE_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,fff,1731769026392.a5db9431e22152bd2a82e84504d2879a. 2024-11-16T15:00:18,099 DEBUG [RS_CLOSE_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 35ef9a7f35bb0b08ec1d53b82d3f9777, disabling compactions & flushes 2024-11-16T15:00:18,099 INFO [RS_CLOSE_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,uuu,1731769026392.35ef9a7f35bb0b08ec1d53b82d3f9777. 2024-11-16T15:00:18,099 DEBUG [RS_CLOSE_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,uuu,1731769026392.35ef9a7f35bb0b08ec1d53b82d3f9777. 2024-11-16T15:00:18,099 DEBUG [RS_CLOSE_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,uuu,1731769026392.35ef9a7f35bb0b08ec1d53b82d3f9777. after waiting 0 ms 2024-11-16T15:00:18,099 DEBUG [RS_CLOSE_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,uuu,1731769026392.35ef9a7f35bb0b08ec1d53b82d3f9777. 2024-11-16T15:00:18,099 INFO [RS_CLOSE_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2902): Flushing 35ef9a7f35bb0b08ec1d53b82d3f9777 3/3 column families, dataSize=32.95 KB heapSize=83.13 KB 2024-11-16T15:00:18,115 DEBUG [RS_CLOSE_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42223/user/jenkins/test-data/a8176cb1-2ebb-b530-b3c7-53c9e53c186e/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/35ef9a7f35bb0b08ec1d53b82d3f9777/.tmp/colfamily11/83d824eda3e24ec69738bdcad6175b24 is 52, key is uuu/colfamily11:colfamily11/1731769030363/Put/seqid=0 2024-11-16T15:00:18,121 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40641 is added to blk_1073741897_1073 (size=39263) 2024-11-16T15:00:18,121 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36531 is added to blk_1073741897_1073 (size=39263) 2024-11-16T15:00:18,121 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41671 is added to blk_1073741897_1073 (size=39263) 2024-11-16T15:00:18,122 INFO [RS_CLOSE_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=32.95 KB at sequenceid=5 (bloomFilter=false), to=hdfs://localhost:42223/user/jenkins/test-data/a8176cb1-2ebb-b530-b3c7-53c9e53c186e/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/35ef9a7f35bb0b08ec1d53b82d3f9777/.tmp/colfamily11/83d824eda3e24ec69738bdcad6175b24 2024-11-16T15:00:18,128 DEBUG [RS_CLOSE_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42223/user/jenkins/test-data/a8176cb1-2ebb-b530-b3c7-53c9e53c186e/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/35ef9a7f35bb0b08ec1d53b82d3f9777/.tmp/colfamily11/83d824eda3e24ec69738bdcad6175b24 as hdfs://localhost:42223/user/jenkins/test-data/a8176cb1-2ebb-b530-b3c7-53c9e53c186e/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/35ef9a7f35bb0b08ec1d53b82d3f9777/colfamily11/83d824eda3e24ec69738bdcad6175b24 2024-11-16T15:00:18,134 INFO [RS_CLOSE_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42223/user/jenkins/test-data/a8176cb1-2ebb-b530-b3c7-53c9e53c186e/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/35ef9a7f35bb0b08ec1d53b82d3f9777/colfamily11/83d824eda3e24ec69738bdcad6175b24, entries=703, sequenceid=5, filesize=38.3 K 2024-11-16T15:00:18,135 INFO [RS_CLOSE_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3140): Finished flush of dataSize ~32.95 KB/33744, heapSize ~82.62 KB/84600, currentSize=0 B/0 for 35ef9a7f35bb0b08ec1d53b82d3f9777 in 36ms, sequenceid=5, compaction requested=false 2024-11-16T15:00:18,139 DEBUG [RS_CLOSE_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:42223/user/jenkins/test-data/a8176cb1-2ebb-b530-b3c7-53c9e53c186e/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/35ef9a7f35bb0b08ec1d53b82d3f9777/recovered.edits/8.seqid, newMaxSeqId=8, maxSeqId=1 2024-11-16T15:00:18,140 INFO [RS_CLOSE_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,uuu,1731769026392.35ef9a7f35bb0b08ec1d53b82d3f9777. 2024-11-16T15:00:18,140 DEBUG [RS_CLOSE_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 35ef9a7f35bb0b08ec1d53b82d3f9777: Waiting for close lock at 1731769218099Running coprocessor pre-close hooks at 1731769218099Disabling compacts and flushes for region at 1731769218099Disabling writes for close at 1731769218099Obtaining lock to block concurrent updates at 1731769218099Preparing flush snapshotting stores in 35ef9a7f35bb0b08ec1d53b82d3f9777 at 1731769218099Finished memstore snapshotting testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,uuu,1731769026392.35ef9a7f35bb0b08ec1d53b82d3f9777., syncing WAL and waiting on mvcc, flushsize=dataSize=33744, getHeapSize=85080, getOffHeapSize=0, getCellsCount=703 at 1731769218099Flushing stores of testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,uuu,1731769026392.35ef9a7f35bb0b08ec1d53b82d3f9777. at 1731769218100 (+1 ms)Flushing 35ef9a7f35bb0b08ec1d53b82d3f9777/colfamily11: creating writer at 1731769218100Flushing 35ef9a7f35bb0b08ec1d53b82d3f9777/colfamily11: appending metadata at 1731769218115 (+15 ms)Flushing 35ef9a7f35bb0b08ec1d53b82d3f9777/colfamily11: closing flushed file at 1731769218115Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@3c7065b7: reopening flushed file at 1731769218127 (+12 ms)Finished flush of dataSize ~32.95 KB/33744, heapSize ~82.62 KB/84600, currentSize=0 B/0 for 35ef9a7f35bb0b08ec1d53b82d3f9777 in 36ms, sequenceid=5, compaction requested=false at 1731769218135 (+8 ms)Writing region close event to WAL at 1731769218135Running coprocessor post-close hooks at 1731769218140 (+5 ms)Closed at 1731769218140 2024-11-16T15:00:18,140 DEBUG [RS_CLOSE_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,uuu,1731769026392.35ef9a7f35bb0b08ec1d53b82d3f9777. 2024-11-16T15:00:18,140 DEBUG [RS_CLOSE_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing fd3944fc2b78246a853fb2232c747c35, disabling compactions & flushes 2024-11-16T15:00:18,140 INFO [RS_CLOSE_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,eee,1731769026392.fd3944fc2b78246a853fb2232c747c35. 2024-11-16T15:00:18,140 DEBUG [RS_CLOSE_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,eee,1731769026392.fd3944fc2b78246a853fb2232c747c35. 2024-11-16T15:00:18,140 DEBUG [RS_CLOSE_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,eee,1731769026392.fd3944fc2b78246a853fb2232c747c35. after waiting 0 ms 2024-11-16T15:00:18,140 DEBUG [RS_CLOSE_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,eee,1731769026392.fd3944fc2b78246a853fb2232c747c35. 2024-11-16T15:00:18,144 DEBUG [RS_CLOSE_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:42223/user/jenkins/test-data/a8176cb1-2ebb-b530-b3c7-53c9e53c186e/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/fd3944fc2b78246a853fb2232c747c35/recovered.edits/7.seqid, newMaxSeqId=7, maxSeqId=4 2024-11-16T15:00:18,145 INFO [RS_CLOSE_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,eee,1731769026392.fd3944fc2b78246a853fb2232c747c35. 2024-11-16T15:00:18,145 DEBUG [RS_CLOSE_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for fd3944fc2b78246a853fb2232c747c35: Waiting for close lock at 1731769218140Running coprocessor pre-close hooks at 1731769218140Disabling compacts and flushes for region at 1731769218140Disabling writes for close at 1731769218140Writing region close event to WAL at 1731769218141 (+1 ms)Running coprocessor post-close hooks at 1731769218145 (+4 ms)Closed at 1731769218145 2024-11-16T15:00:18,145 DEBUG [RS_CLOSE_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,eee,1731769026392.fd3944fc2b78246a853fb2232c747c35. 2024-11-16T15:00:18,145 DEBUG [RS_CLOSE_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing c3dde5f5f668d914608a2987d4f4c001, disabling compactions & flushes 2024-11-16T15:00:18,145 INFO [RS_CLOSE_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,bbb,1731769026392.c3dde5f5f668d914608a2987d4f4c001. 2024-11-16T15:00:18,145 DEBUG [RS_CLOSE_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,bbb,1731769026392.c3dde5f5f668d914608a2987d4f4c001. 2024-11-16T15:00:18,145 DEBUG [RS_CLOSE_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,bbb,1731769026392.c3dde5f5f668d914608a2987d4f4c001. after waiting 0 ms 2024-11-16T15:00:18,145 DEBUG [RS_CLOSE_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,bbb,1731769026392.c3dde5f5f668d914608a2987d4f4c001. 2024-11-16T15:00:18,145 INFO [RS_CLOSE_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2902): Flushing c3dde5f5f668d914608a2987d4f4c001 3/3 column families, dataSize=32.95 KB heapSize=83.13 KB 2024-11-16T15:00:18,160 DEBUG [RS_CLOSE_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42223/user/jenkins/test-data/a8176cb1-2ebb-b530-b3c7-53c9e53c186e/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/c3dde5f5f668d914608a2987d4f4c001/.tmp/colfamily11/a2db626471ed4ce4a781b6ff772b91b2 is 52, key is bbb/colfamily11:colfamily11/1731769030022/Put/seqid=0 2024-11-16T15:00:18,166 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36531 is added to blk_1073741898_1074 (size=39263) 2024-11-16T15:00:18,166 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41671 is added to blk_1073741898_1074 (size=39263) 2024-11-16T15:00:18,166 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40641 is added to blk_1073741898_1074 (size=39263) 2024-11-16T15:00:18,167 INFO [RS_CLOSE_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=32.95 KB at sequenceid=5 (bloomFilter=false), to=hdfs://localhost:42223/user/jenkins/test-data/a8176cb1-2ebb-b530-b3c7-53c9e53c186e/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/c3dde5f5f668d914608a2987d4f4c001/.tmp/colfamily11/a2db626471ed4ce4a781b6ff772b91b2 2024-11-16T15:00:18,173 DEBUG [RS_CLOSE_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42223/user/jenkins/test-data/a8176cb1-2ebb-b530-b3c7-53c9e53c186e/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/c3dde5f5f668d914608a2987d4f4c001/.tmp/colfamily11/a2db626471ed4ce4a781b6ff772b91b2 as hdfs://localhost:42223/user/jenkins/test-data/a8176cb1-2ebb-b530-b3c7-53c9e53c186e/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/c3dde5f5f668d914608a2987d4f4c001/colfamily11/a2db626471ed4ce4a781b6ff772b91b2 2024-11-16T15:00:18,179 INFO [RS_CLOSE_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42223/user/jenkins/test-data/a8176cb1-2ebb-b530-b3c7-53c9e53c186e/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/c3dde5f5f668d914608a2987d4f4c001/colfamily11/a2db626471ed4ce4a781b6ff772b91b2, entries=703, sequenceid=5, filesize=38.3 K 2024-11-16T15:00:18,180 INFO [RS_CLOSE_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3140): Finished flush of dataSize ~32.95 KB/33744, heapSize ~82.62 KB/84600, currentSize=0 B/0 for c3dde5f5f668d914608a2987d4f4c001 in 35ms, sequenceid=5, compaction requested=false 2024-11-16T15:00:18,184 DEBUG [RS_CLOSE_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:42223/user/jenkins/test-data/a8176cb1-2ebb-b530-b3c7-53c9e53c186e/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/c3dde5f5f668d914608a2987d4f4c001/recovered.edits/8.seqid, newMaxSeqId=8, maxSeqId=1 2024-11-16T15:00:18,185 INFO [RS_CLOSE_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,bbb,1731769026392.c3dde5f5f668d914608a2987d4f4c001. 2024-11-16T15:00:18,185 DEBUG [RS_CLOSE_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for c3dde5f5f668d914608a2987d4f4c001: Waiting for close lock at 1731769218145Running coprocessor pre-close hooks at 1731769218145Disabling compacts and flushes for region at 1731769218145Disabling writes for close at 1731769218145Obtaining lock to block concurrent updates at 1731769218145Preparing flush snapshotting stores in c3dde5f5f668d914608a2987d4f4c001 at 1731769218145Finished memstore snapshotting testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,bbb,1731769026392.c3dde5f5f668d914608a2987d4f4c001., syncing WAL and waiting on mvcc, flushsize=dataSize=33744, getHeapSize=85080, getOffHeapSize=0, getCellsCount=703 at 1731769218146 (+1 ms)Flushing stores of testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,bbb,1731769026392.c3dde5f5f668d914608a2987d4f4c001. at 1731769218146Flushing c3dde5f5f668d914608a2987d4f4c001/colfamily11: creating writer at 1731769218146Flushing c3dde5f5f668d914608a2987d4f4c001/colfamily11: appending metadata at 1731769218160 (+14 ms)Flushing c3dde5f5f668d914608a2987d4f4c001/colfamily11: closing flushed file at 1731769218160Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@12b83f7e: reopening flushed file at 1731769218172 (+12 ms)Finished flush of dataSize ~32.95 KB/33744, heapSize ~82.62 KB/84600, currentSize=0 B/0 for c3dde5f5f668d914608a2987d4f4c001 in 35ms, sequenceid=5, compaction requested=false at 1731769218181 (+9 ms)Writing region close event to WAL at 1731769218181Running coprocessor post-close hooks at 1731769218185 (+4 ms)Closed at 1731769218185 2024-11-16T15:00:18,185 DEBUG [RS_CLOSE_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,bbb,1731769026392.c3dde5f5f668d914608a2987d4f4c001. 2024-11-16T15:00:18,185 DEBUG [RS_CLOSE_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 7eb7b75d28b808c5c3f2f96e407b4347, disabling compactions & flushes 2024-11-16T15:00:18,185 INFO [RS_CLOSE_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,jjj,1731769026392.7eb7b75d28b808c5c3f2f96e407b4347. 2024-11-16T15:00:18,185 DEBUG [RS_CLOSE_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,jjj,1731769026392.7eb7b75d28b808c5c3f2f96e407b4347. 2024-11-16T15:00:18,185 DEBUG [RS_CLOSE_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,jjj,1731769026392.7eb7b75d28b808c5c3f2f96e407b4347. after waiting 0 ms 2024-11-16T15:00:18,185 DEBUG [RS_CLOSE_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,jjj,1731769026392.7eb7b75d28b808c5c3f2f96e407b4347. 2024-11-16T15:00:18,189 DEBUG [RS_CLOSE_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:42223/user/jenkins/test-data/a8176cb1-2ebb-b530-b3c7-53c9e53c186e/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/7eb7b75d28b808c5c3f2f96e407b4347/recovered.edits/7.seqid, newMaxSeqId=7, maxSeqId=4 2024-11-16T15:00:18,190 INFO [RS_CLOSE_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,jjj,1731769026392.7eb7b75d28b808c5c3f2f96e407b4347. 2024-11-16T15:00:18,190 DEBUG [RS_CLOSE_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 7eb7b75d28b808c5c3f2f96e407b4347: Waiting for close lock at 1731769218185Running coprocessor pre-close hooks at 1731769218185Disabling compacts and flushes for region at 1731769218185Disabling writes for close at 1731769218185Writing region close event to WAL at 1731769218186 (+1 ms)Running coprocessor post-close hooks at 1731769218189 (+3 ms)Closed at 1731769218189 2024-11-16T15:00:18,190 DEBUG [RS_CLOSE_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,jjj,1731769026392.7eb7b75d28b808c5c3f2f96e407b4347. 2024-11-16T15:00:18,190 DEBUG [RS_CLOSE_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing dcd6414ddebad66e87ab37a6dba924f3, disabling compactions & flushes 2024-11-16T15:00:18,190 INFO [RS_CLOSE_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ggg,1731769026392.dcd6414ddebad66e87ab37a6dba924f3. 2024-11-16T15:00:18,190 DEBUG [RS_CLOSE_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ggg,1731769026392.dcd6414ddebad66e87ab37a6dba924f3. 2024-11-16T15:00:18,190 DEBUG [RS_CLOSE_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ggg,1731769026392.dcd6414ddebad66e87ab37a6dba924f3. after waiting 0 ms 2024-11-16T15:00:18,190 DEBUG [RS_CLOSE_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ggg,1731769026392.dcd6414ddebad66e87ab37a6dba924f3. 2024-11-16T15:00:18,190 INFO [RS_CLOSE_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2902): Flushing dcd6414ddebad66e87ab37a6dba924f3 3/3 column families, dataSize=32.95 KB heapSize=83.13 KB 2024-11-16T15:00:18,206 DEBUG [RS_CLOSE_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42223/user/jenkins/test-data/a8176cb1-2ebb-b530-b3c7-53c9e53c186e/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/dcd6414ddebad66e87ab37a6dba924f3/.tmp/colfamily11/61e39275aa124ff695b844bae9b0cb94 is 52, key is ggg/colfamily11:colfamily11/1731769030153/Put/seqid=0 2024-11-16T15:00:18,211 DEBUG [RS:2;fc3fda29f660:42985 {}] regionserver.HRegionServer(1351): Waiting on 1d97df67e5110b0fe1a1e4b7c1341cbb, 266565e60d7419a2a30ceec1b0f6519b, 299131947f15f9c44bcfe22cd6623096, 7f5c9df3a544933227f58b052f743f72, b364a006589fe0d2474ffa45692a6efc, dc4fcaece350cd66f6bf88cfbdff6f15, dcd6414ddebad66e87ab37a6dba924f3 2024-11-16T15:00:18,211 INFO [RS:0;fc3fda29f660:39089 {}] regionserver.HRegionServer(976): stopping server fc3fda29f660,39089,1731769023769; all regions closed. 2024-11-16T15:00:18,212 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41671 is added to blk_1073741899_1075 (size=39263) 2024-11-16T15:00:18,212 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40641 is added to blk_1073741899_1075 (size=39263) 2024-11-16T15:00:18,212 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36531 is added to blk_1073741899_1075 (size=39263) 2024-11-16T15:00:18,213 INFO [RS_CLOSE_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=32.95 KB at sequenceid=5 (bloomFilter=false), to=hdfs://localhost:42223/user/jenkins/test-data/a8176cb1-2ebb-b530-b3c7-53c9e53c186e/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/dcd6414ddebad66e87ab37a6dba924f3/.tmp/colfamily11/61e39275aa124ff695b844bae9b0cb94 2024-11-16T15:00:18,214 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41671 is added to blk_1073741865_1041 (size=15722) 2024-11-16T15:00:18,214 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36531 is added to blk_1073741865_1041 (size=15722) 2024-11-16T15:00:18,214 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40641 is added to blk_1073741865_1041 (size=15722) 2024-11-16T15:00:18,217 DEBUG [RS:0;fc3fda29f660:39089 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/a8176cb1-2ebb-b530-b3c7-53c9e53c186e/oldWALs 2024-11-16T15:00:18,217 INFO [RS:0;fc3fda29f660:39089 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL fc3fda29f660%2C39089%2C1731769023769.meta:.meta(num 1731769212069) 2024-11-16T15:00:18,220 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40641 is added to blk_1073741835_1011 (size=278891) 2024-11-16T15:00:18,220 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36531 is added to blk_1073741835_1011 (size=278891) 2024-11-16T15:00:18,220 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41671 is added to blk_1073741835_1011 (size=278891) 2024-11-16T15:00:18,220 DEBUG [RS_CLOSE_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42223/user/jenkins/test-data/a8176cb1-2ebb-b530-b3c7-53c9e53c186e/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/dcd6414ddebad66e87ab37a6dba924f3/.tmp/colfamily11/61e39275aa124ff695b844bae9b0cb94 as hdfs://localhost:42223/user/jenkins/test-data/a8176cb1-2ebb-b530-b3c7-53c9e53c186e/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/dcd6414ddebad66e87ab37a6dba924f3/colfamily11/61e39275aa124ff695b844bae9b0cb94 2024-11-16T15:00:18,223 DEBUG [RS:0;fc3fda29f660:39089 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/a8176cb1-2ebb-b530-b3c7-53c9e53c186e/oldWALs 2024-11-16T15:00:18,223 INFO [RS:0;fc3fda29f660:39089 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL fc3fda29f660%2C39089%2C1731769023769:(num 1731769025435) 2024-11-16T15:00:18,223 DEBUG [RS:0;fc3fda29f660:39089 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T15:00:18,223 INFO [RS:0;fc3fda29f660:39089 {}] regionserver.LeaseManager(133): Closed leases 2024-11-16T15:00:18,223 INFO [RS:0;fc3fda29f660:39089 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-16T15:00:18,223 INFO [RS:0;fc3fda29f660:39089 {}] hbase.ChoreService(370): Chore service for: regionserver/fc3fda29f660:0 had [ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-11-16T15:00:18,224 INFO [RS:0;fc3fda29f660:39089 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-16T15:00:18,224 INFO [regionserver/fc3fda29f660:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-16T15:00:18,224 INFO [RS:0;fc3fda29f660:39089 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:39089 2024-11-16T15:00:18,227 INFO [RS_CLOSE_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42223/user/jenkins/test-data/a8176cb1-2ebb-b530-b3c7-53c9e53c186e/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/dcd6414ddebad66e87ab37a6dba924f3/colfamily11/61e39275aa124ff695b844bae9b0cb94, entries=703, sequenceid=5, filesize=38.3 K 2024-11-16T15:00:18,228 INFO [RS_CLOSE_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3140): Finished flush of dataSize ~32.95 KB/33744, heapSize ~82.62 KB/84600, currentSize=0 B/0 for dcd6414ddebad66e87ab37a6dba924f3 in 38ms, sequenceid=5, compaction requested=false 2024-11-16T15:00:18,233 DEBUG [RS_CLOSE_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:42223/user/jenkins/test-data/a8176cb1-2ebb-b530-b3c7-53c9e53c186e/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/dcd6414ddebad66e87ab37a6dba924f3/recovered.edits/8.seqid, newMaxSeqId=8, maxSeqId=1 2024-11-16T15:00:18,233 INFO [RS_CLOSE_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ggg,1731769026392.dcd6414ddebad66e87ab37a6dba924f3. 2024-11-16T15:00:18,233 DEBUG [RS_CLOSE_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for dcd6414ddebad66e87ab37a6dba924f3: Waiting for close lock at 1731769218190Running coprocessor pre-close hooks at 1731769218190Disabling compacts and flushes for region at 1731769218190Disabling writes for close at 1731769218190Obtaining lock to block concurrent updates at 1731769218190Preparing flush snapshotting stores in dcd6414ddebad66e87ab37a6dba924f3 at 1731769218190Finished memstore snapshotting testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ggg,1731769026392.dcd6414ddebad66e87ab37a6dba924f3., syncing WAL and waiting on mvcc, flushsize=dataSize=33744, getHeapSize=85080, getOffHeapSize=0, getCellsCount=703 at 1731769218190Flushing stores of testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ggg,1731769026392.dcd6414ddebad66e87ab37a6dba924f3. at 1731769218191 (+1 ms)Flushing dcd6414ddebad66e87ab37a6dba924f3/colfamily11: creating writer at 1731769218191Flushing dcd6414ddebad66e87ab37a6dba924f3/colfamily11: appending metadata at 1731769218206 (+15 ms)Flushing dcd6414ddebad66e87ab37a6dba924f3/colfamily11: closing flushed file at 1731769218206Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@67bc1ab8: reopening flushed file at 1731769218219 (+13 ms)Finished flush of dataSize ~32.95 KB/33744, heapSize ~82.62 KB/84600, currentSize=0 B/0 for dcd6414ddebad66e87ab37a6dba924f3 in 38ms, sequenceid=5, compaction requested=false at 1731769218228 (+9 ms)Writing region close event to WAL at 1731769218229 (+1 ms)Running coprocessor post-close hooks at 1731769218233 (+4 ms)Closed at 1731769218233 2024-11-16T15:00:18,234 DEBUG [RS_CLOSE_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ggg,1731769026392.dcd6414ddebad66e87ab37a6dba924f3. 2024-11-16T15:00:18,234 DEBUG [RS_CLOSE_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing dc4fcaece350cd66f6bf88cfbdff6f15, disabling compactions & flushes 2024-11-16T15:00:18,234 INFO [RS_CLOSE_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,xxx,1731769026392.dc4fcaece350cd66f6bf88cfbdff6f15. 2024-11-16T15:00:18,234 DEBUG [RS_CLOSE_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,xxx,1731769026392.dc4fcaece350cd66f6bf88cfbdff6f15. 2024-11-16T15:00:18,234 DEBUG [RS_CLOSE_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,xxx,1731769026392.dc4fcaece350cd66f6bf88cfbdff6f15. after waiting 0 ms 2024-11-16T15:00:18,234 DEBUG [RS_CLOSE_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,xxx,1731769026392.dc4fcaece350cd66f6bf88cfbdff6f15. 2024-11-16T15:00:18,234 INFO [RS_CLOSE_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2902): Flushing dc4fcaece350cd66f6bf88cfbdff6f15 3/3 column families, dataSize=32.95 KB heapSize=83.13 KB 2024-11-16T15:00:18,255 DEBUG [RS_CLOSE_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42223/user/jenkins/test-data/a8176cb1-2ebb-b530-b3c7-53c9e53c186e/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/dc4fcaece350cd66f6bf88cfbdff6f15/.tmp/colfamily11/2383f0908f3045349ba15cfc92b03c62 is 52, key is xxx/colfamily11:colfamily11/1731769030413/Put/seqid=0 2024-11-16T15:00:18,262 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40641 is added to blk_1073741900_1076 (size=39263) 2024-11-16T15:00:18,262 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41671 is added to blk_1073741900_1076 (size=39263) 2024-11-16T15:00:18,262 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36531 is added to blk_1073741900_1076 (size=39263) 2024-11-16T15:00:18,263 INFO [RS_CLOSE_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=32.95 KB at sequenceid=5 (bloomFilter=false), to=hdfs://localhost:42223/user/jenkins/test-data/a8176cb1-2ebb-b530-b3c7-53c9e53c186e/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/dc4fcaece350cd66f6bf88cfbdff6f15/.tmp/colfamily11/2383f0908f3045349ba15cfc92b03c62 2024-11-16T15:00:18,269 DEBUG [RS_CLOSE_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42223/user/jenkins/test-data/a8176cb1-2ebb-b530-b3c7-53c9e53c186e/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/dc4fcaece350cd66f6bf88cfbdff6f15/.tmp/colfamily11/2383f0908f3045349ba15cfc92b03c62 as hdfs://localhost:42223/user/jenkins/test-data/a8176cb1-2ebb-b530-b3c7-53c9e53c186e/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/dc4fcaece350cd66f6bf88cfbdff6f15/colfamily11/2383f0908f3045349ba15cfc92b03c62 2024-11-16T15:00:18,269 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43615-0x101456fc9c70000, quorum=127.0.0.1:63224, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-16T15:00:18,269 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39089-0x101456fc9c70001, quorum=127.0.0.1:63224, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/fc3fda29f660,39089,1731769023769 2024-11-16T15:00:18,270 INFO [RS:0;fc3fda29f660:39089 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-16T15:00:18,270 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [fc3fda29f660,39089,1731769023769] 2024-11-16T15:00:18,275 INFO [RS_CLOSE_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42223/user/jenkins/test-data/a8176cb1-2ebb-b530-b3c7-53c9e53c186e/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/dc4fcaece350cd66f6bf88cfbdff6f15/colfamily11/2383f0908f3045349ba15cfc92b03c62, entries=703, sequenceid=5, filesize=38.3 K 2024-11-16T15:00:18,276 INFO [RS_CLOSE_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3140): Finished flush of dataSize ~32.95 KB/33744, heapSize ~82.62 KB/84600, currentSize=0 B/0 for dc4fcaece350cd66f6bf88cfbdff6f15 in 42ms, sequenceid=5, compaction requested=false 2024-11-16T15:00:18,279 INFO [regionserver/fc3fda29f660:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: MemstoreFlusherChore was stopped 2024-11-16T15:00:18,279 INFO [regionserver/fc3fda29f660:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: CompactionChecker was stopped 2024-11-16T15:00:18,280 DEBUG [RS_CLOSE_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:42223/user/jenkins/test-data/a8176cb1-2ebb-b530-b3c7-53c9e53c186e/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/dc4fcaece350cd66f6bf88cfbdff6f15/recovered.edits/8.seqid, newMaxSeqId=8, maxSeqId=1 2024-11-16T15:00:18,280 INFO [RS_CLOSE_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,xxx,1731769026392.dc4fcaece350cd66f6bf88cfbdff6f15. 2024-11-16T15:00:18,280 DEBUG [RS_CLOSE_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for dc4fcaece350cd66f6bf88cfbdff6f15: Waiting for close lock at 1731769218234Running coprocessor pre-close hooks at 1731769218234Disabling compacts and flushes for region at 1731769218234Disabling writes for close at 1731769218234Obtaining lock to block concurrent updates at 1731769218234Preparing flush snapshotting stores in dc4fcaece350cd66f6bf88cfbdff6f15 at 1731769218234Finished memstore snapshotting testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,xxx,1731769026392.dc4fcaece350cd66f6bf88cfbdff6f15., syncing WAL and waiting on mvcc, flushsize=dataSize=33744, getHeapSize=85080, getOffHeapSize=0, getCellsCount=703 at 1731769218234Flushing stores of testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,xxx,1731769026392.dc4fcaece350cd66f6bf88cfbdff6f15. at 1731769218235 (+1 ms)Flushing dc4fcaece350cd66f6bf88cfbdff6f15/colfamily11: creating writer at 1731769218235Flushing dc4fcaece350cd66f6bf88cfbdff6f15/colfamily11: appending metadata at 1731769218255 (+20 ms)Flushing dc4fcaece350cd66f6bf88cfbdff6f15/colfamily11: closing flushed file at 1731769218255Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@658091c3: reopening flushed file at 1731769218268 (+13 ms)Finished flush of dataSize ~32.95 KB/33744, heapSize ~82.62 KB/84600, currentSize=0 B/0 for dc4fcaece350cd66f6bf88cfbdff6f15 in 42ms, sequenceid=5, compaction requested=false at 1731769218276 (+8 ms)Writing region close event to WAL at 1731769218277 (+1 ms)Running coprocessor post-close hooks at 1731769218280 (+3 ms)Closed at 1731769218280 2024-11-16T15:00:18,281 DEBUG [RS_CLOSE_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,xxx,1731769026392.dc4fcaece350cd66f6bf88cfbdff6f15. 2024-11-16T15:00:18,281 DEBUG [RS_CLOSE_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 7f5c9df3a544933227f58b052f743f72, disabling compactions & flushes 2024-11-16T15:00:18,281 INFO [RS_CLOSE_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ppp,1731769026392.7f5c9df3a544933227f58b052f743f72. 2024-11-16T15:00:18,281 DEBUG [RS_CLOSE_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ppp,1731769026392.7f5c9df3a544933227f58b052f743f72. 2024-11-16T15:00:18,281 DEBUG [RS_CLOSE_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ppp,1731769026392.7f5c9df3a544933227f58b052f743f72. after waiting 0 ms 2024-11-16T15:00:18,281 DEBUG [RS_CLOSE_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ppp,1731769026392.7f5c9df3a544933227f58b052f743f72. 2024-11-16T15:00:18,284 DEBUG [RS_CLOSE_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:42223/user/jenkins/test-data/a8176cb1-2ebb-b530-b3c7-53c9e53c186e/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/7f5c9df3a544933227f58b052f743f72/recovered.edits/7.seqid, newMaxSeqId=7, maxSeqId=4 2024-11-16T15:00:18,285 INFO [RS_CLOSE_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ppp,1731769026392.7f5c9df3a544933227f58b052f743f72. 2024-11-16T15:00:18,285 DEBUG [RS_CLOSE_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 7f5c9df3a544933227f58b052f743f72: Waiting for close lock at 1731769218281Running coprocessor pre-close hooks at 1731769218281Disabling compacts and flushes for region at 1731769218281Disabling writes for close at 1731769218281Writing region close event to WAL at 1731769218281Running coprocessor post-close hooks at 1731769218285 (+4 ms)Closed at 1731769218285 2024-11-16T15:00:18,285 DEBUG [RS_CLOSE_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ppp,1731769026392.7f5c9df3a544933227f58b052f743f72. 2024-11-16T15:00:18,285 DEBUG [RS_CLOSE_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 1d97df67e5110b0fe1a1e4b7c1341cbb, disabling compactions & flushes 2024-11-16T15:00:18,285 INFO [RS_CLOSE_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,hhh,1731769026392.1d97df67e5110b0fe1a1e4b7c1341cbb. 2024-11-16T15:00:18,285 DEBUG [RS_CLOSE_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,hhh,1731769026392.1d97df67e5110b0fe1a1e4b7c1341cbb. 2024-11-16T15:00:18,285 DEBUG [RS_CLOSE_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,hhh,1731769026392.1d97df67e5110b0fe1a1e4b7c1341cbb. after waiting 0 ms 2024-11-16T15:00:18,286 DEBUG [RS_CLOSE_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,hhh,1731769026392.1d97df67e5110b0fe1a1e4b7c1341cbb. 2024-11-16T15:00:18,286 INFO [RS_CLOSE_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2902): Flushing 1d97df67e5110b0fe1a1e4b7c1341cbb 3/3 column families, dataSize=32.95 KB heapSize=83.13 KB 2024-11-16T15:00:18,286 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/fc3fda29f660,39089,1731769023769 already deleted, retry=false 2024-11-16T15:00:18,286 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; fc3fda29f660,39089,1731769023769 expired; onlineServers=1 2024-11-16T15:00:18,300 DEBUG [RS_CLOSE_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42223/user/jenkins/test-data/a8176cb1-2ebb-b530-b3c7-53c9e53c186e/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/1d97df67e5110b0fe1a1e4b7c1341cbb/.tmp/colfamily11/ac64f10f026442f381444a14237fac8f is 52, key is hhh/colfamily11:colfamily11/1731769030204/Put/seqid=0 2024-11-16T15:00:18,307 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40641 is added to blk_1073741901_1077 (size=39263) 2024-11-16T15:00:18,307 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36531 is added to blk_1073741901_1077 (size=39263) 2024-11-16T15:00:18,307 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41671 is added to blk_1073741901_1077 (size=39263) 2024-11-16T15:00:18,308 INFO [RS_CLOSE_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=32.95 KB at sequenceid=5 (bloomFilter=false), to=hdfs://localhost:42223/user/jenkins/test-data/a8176cb1-2ebb-b530-b3c7-53c9e53c186e/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/1d97df67e5110b0fe1a1e4b7c1341cbb/.tmp/colfamily11/ac64f10f026442f381444a14237fac8f 2024-11-16T15:00:18,315 DEBUG [RS_CLOSE_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42223/user/jenkins/test-data/a8176cb1-2ebb-b530-b3c7-53c9e53c186e/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/1d97df67e5110b0fe1a1e4b7c1341cbb/.tmp/colfamily11/ac64f10f026442f381444a14237fac8f as hdfs://localhost:42223/user/jenkins/test-data/a8176cb1-2ebb-b530-b3c7-53c9e53c186e/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/1d97df67e5110b0fe1a1e4b7c1341cbb/colfamily11/ac64f10f026442f381444a14237fac8f 2024-11-16T15:00:18,322 INFO [RS_CLOSE_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42223/user/jenkins/test-data/a8176cb1-2ebb-b530-b3c7-53c9e53c186e/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/1d97df67e5110b0fe1a1e4b7c1341cbb/colfamily11/ac64f10f026442f381444a14237fac8f, entries=703, sequenceid=5, filesize=38.3 K 2024-11-16T15:00:18,323 INFO [RS_CLOSE_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3140): Finished flush of dataSize ~32.95 KB/33744, heapSize ~82.62 KB/84600, currentSize=0 B/0 for 1d97df67e5110b0fe1a1e4b7c1341cbb in 36ms, sequenceid=5, compaction requested=false 2024-11-16T15:00:18,327 DEBUG [RS_CLOSE_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:42223/user/jenkins/test-data/a8176cb1-2ebb-b530-b3c7-53c9e53c186e/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/1d97df67e5110b0fe1a1e4b7c1341cbb/recovered.edits/8.seqid, newMaxSeqId=8, maxSeqId=1 2024-11-16T15:00:18,327 INFO [RS_CLOSE_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,hhh,1731769026392.1d97df67e5110b0fe1a1e4b7c1341cbb. 2024-11-16T15:00:18,327 DEBUG [RS_CLOSE_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 1d97df67e5110b0fe1a1e4b7c1341cbb: Waiting for close lock at 1731769218285Running coprocessor pre-close hooks at 1731769218285Disabling compacts and flushes for region at 1731769218285Disabling writes for close at 1731769218286 (+1 ms)Obtaining lock to block concurrent updates at 1731769218286Preparing flush snapshotting stores in 1d97df67e5110b0fe1a1e4b7c1341cbb at 1731769218286Finished memstore snapshotting testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,hhh,1731769026392.1d97df67e5110b0fe1a1e4b7c1341cbb., syncing WAL and waiting on mvcc, flushsize=dataSize=33744, getHeapSize=85080, getOffHeapSize=0, getCellsCount=703 at 1731769218286Flushing stores of testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,hhh,1731769026392.1d97df67e5110b0fe1a1e4b7c1341cbb. at 1731769218286Flushing 1d97df67e5110b0fe1a1e4b7c1341cbb/colfamily11: creating writer at 1731769218286Flushing 1d97df67e5110b0fe1a1e4b7c1341cbb/colfamily11: appending metadata at 1731769218300 (+14 ms)Flushing 1d97df67e5110b0fe1a1e4b7c1341cbb/colfamily11: closing flushed file at 1731769218300Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@3e0807bc: reopening flushed file at 1731769218314 (+14 ms)Finished flush of dataSize ~32.95 KB/33744, heapSize ~82.62 KB/84600, currentSize=0 B/0 for 1d97df67e5110b0fe1a1e4b7c1341cbb in 36ms, sequenceid=5, compaction requested=false at 1731769218323 (+9 ms)Writing region close event to WAL at 1731769218323Running coprocessor post-close hooks at 1731769218327 (+4 ms)Closed at 1731769218327 2024-11-16T15:00:18,328 DEBUG [RS_CLOSE_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,hhh,1731769026392.1d97df67e5110b0fe1a1e4b7c1341cbb. 2024-11-16T15:00:18,328 DEBUG [RS_CLOSE_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 299131947f15f9c44bcfe22cd6623096, disabling compactions & flushes 2024-11-16T15:00:18,328 INFO [RS_CLOSE_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ooo,1731769026392.299131947f15f9c44bcfe22cd6623096. 2024-11-16T15:00:18,328 DEBUG [RS_CLOSE_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ooo,1731769026392.299131947f15f9c44bcfe22cd6623096. 2024-11-16T15:00:18,328 DEBUG [RS_CLOSE_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ooo,1731769026392.299131947f15f9c44bcfe22cd6623096. after waiting 0 ms 2024-11-16T15:00:18,328 DEBUG [RS_CLOSE_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ooo,1731769026392.299131947f15f9c44bcfe22cd6623096. 2024-11-16T15:00:18,328 INFO [RS_CLOSE_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2902): Flushing 299131947f15f9c44bcfe22cd6623096 3/3 column families, dataSize=32.95 KB heapSize=83.13 KB 2024-11-16T15:00:18,343 DEBUG [RS_CLOSE_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42223/user/jenkins/test-data/a8176cb1-2ebb-b530-b3c7-53c9e53c186e/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/299131947f15f9c44bcfe22cd6623096/.tmp/colfamily11/764a148e06dc494b8f4c673022fc175f is 52, key is ooo/colfamily11:colfamily11/1731769030269/Put/seqid=0 2024-11-16T15:00:18,349 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41671 is added to blk_1073741902_1078 (size=39263) 2024-11-16T15:00:18,350 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36531 is added to blk_1073741902_1078 (size=39263) 2024-11-16T15:00:18,350 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40641 is added to blk_1073741902_1078 (size=39263) 2024-11-16T15:00:18,350 INFO [RS_CLOSE_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=32.95 KB at sequenceid=5 (bloomFilter=false), to=hdfs://localhost:42223/user/jenkins/test-data/a8176cb1-2ebb-b530-b3c7-53c9e53c186e/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/299131947f15f9c44bcfe22cd6623096/.tmp/colfamily11/764a148e06dc494b8f4c673022fc175f 2024-11-16T15:00:18,357 DEBUG [RS_CLOSE_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42223/user/jenkins/test-data/a8176cb1-2ebb-b530-b3c7-53c9e53c186e/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/299131947f15f9c44bcfe22cd6623096/.tmp/colfamily11/764a148e06dc494b8f4c673022fc175f as hdfs://localhost:42223/user/jenkins/test-data/a8176cb1-2ebb-b530-b3c7-53c9e53c186e/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/299131947f15f9c44bcfe22cd6623096/colfamily11/764a148e06dc494b8f4c673022fc175f 2024-11-16T15:00:18,363 INFO [RS_CLOSE_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42223/user/jenkins/test-data/a8176cb1-2ebb-b530-b3c7-53c9e53c186e/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/299131947f15f9c44bcfe22cd6623096/colfamily11/764a148e06dc494b8f4c673022fc175f, entries=703, sequenceid=5, filesize=38.3 K 2024-11-16T15:00:18,364 INFO [RS_CLOSE_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3140): Finished flush of dataSize ~32.95 KB/33744, heapSize ~82.62 KB/84600, currentSize=0 B/0 for 299131947f15f9c44bcfe22cd6623096 in 36ms, sequenceid=5, compaction requested=false 2024-11-16T15:00:18,368 DEBUG [RS_CLOSE_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:42223/user/jenkins/test-data/a8176cb1-2ebb-b530-b3c7-53c9e53c186e/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/299131947f15f9c44bcfe22cd6623096/recovered.edits/8.seqid, newMaxSeqId=8, maxSeqId=1 2024-11-16T15:00:18,369 INFO [RS_CLOSE_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ooo,1731769026392.299131947f15f9c44bcfe22cd6623096. 2024-11-16T15:00:18,369 DEBUG [RS_CLOSE_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 299131947f15f9c44bcfe22cd6623096: Waiting for close lock at 1731769218328Running coprocessor pre-close hooks at 1731769218328Disabling compacts and flushes for region at 1731769218328Disabling writes for close at 1731769218328Obtaining lock to block concurrent updates at 1731769218328Preparing flush snapshotting stores in 299131947f15f9c44bcfe22cd6623096 at 1731769218328Finished memstore snapshotting testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ooo,1731769026392.299131947f15f9c44bcfe22cd6623096., syncing WAL and waiting on mvcc, flushsize=dataSize=33744, getHeapSize=85080, getOffHeapSize=0, getCellsCount=703 at 1731769218328Flushing stores of testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ooo,1731769026392.299131947f15f9c44bcfe22cd6623096. at 1731769218329 (+1 ms)Flushing 299131947f15f9c44bcfe22cd6623096/colfamily11: creating writer at 1731769218329Flushing 299131947f15f9c44bcfe22cd6623096/colfamily11: appending metadata at 1731769218343 (+14 ms)Flushing 299131947f15f9c44bcfe22cd6623096/colfamily11: closing flushed file at 1731769218343Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@31fb6eb4: reopening flushed file at 1731769218356 (+13 ms)Finished flush of dataSize ~32.95 KB/33744, heapSize ~82.62 KB/84600, currentSize=0 B/0 for 299131947f15f9c44bcfe22cd6623096 in 36ms, sequenceid=5, compaction requested=false at 1731769218364 (+8 ms)Writing region close event to WAL at 1731769218365 (+1 ms)Running coprocessor post-close hooks at 1731769218369 (+4 ms)Closed at 1731769218369 2024-11-16T15:00:18,369 DEBUG [RS_CLOSE_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ooo,1731769026392.299131947f15f9c44bcfe22cd6623096. 2024-11-16T15:00:18,369 DEBUG [RS_CLOSE_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing b364a006589fe0d2474ffa45692a6efc, disabling compactions & flushes 2024-11-16T15:00:18,369 INFO [RS_CLOSE_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,rrr,1731769026392.b364a006589fe0d2474ffa45692a6efc. 2024-11-16T15:00:18,369 DEBUG [RS_CLOSE_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,rrr,1731769026392.b364a006589fe0d2474ffa45692a6efc. 2024-11-16T15:00:18,369 DEBUG [RS_CLOSE_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,rrr,1731769026392.b364a006589fe0d2474ffa45692a6efc. after waiting 0 ms 2024-11-16T15:00:18,369 DEBUG [RS_CLOSE_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,rrr,1731769026392.b364a006589fe0d2474ffa45692a6efc. 2024-11-16T15:00:18,369 INFO [RS_CLOSE_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2902): Flushing b364a006589fe0d2474ffa45692a6efc 3/3 column families, dataSize=32.95 KB heapSize=83.13 KB 2024-11-16T15:00:18,378 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39089-0x101456fc9c70001, quorum=127.0.0.1:63224, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-16T15:00:18,378 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39089-0x101456fc9c70001, quorum=127.0.0.1:63224, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-16T15:00:18,378 INFO [RS:0;fc3fda29f660:39089 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-16T15:00:18,378 INFO [RS:0;fc3fda29f660:39089 {}] regionserver.HRegionServer(1031): Exiting; stopping=fc3fda29f660,39089,1731769023769; zookeeper connection closed. 2024-11-16T15:00:18,378 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@1439d601 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@1439d601 2024-11-16T15:00:18,385 DEBUG [RS_CLOSE_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42223/user/jenkins/test-data/a8176cb1-2ebb-b530-b3c7-53c9e53c186e/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/b364a006589fe0d2474ffa45692a6efc/.tmp/colfamily11/947260ab3ca148909c71e7953870e83f is 52, key is rrr/colfamily11:colfamily11/1731769030323/Put/seqid=0 2024-11-16T15:00:18,391 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41671 is added to blk_1073741903_1079 (size=39263) 2024-11-16T15:00:18,391 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40641 is added to blk_1073741903_1079 (size=39263) 2024-11-16T15:00:18,392 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36531 is added to blk_1073741903_1079 (size=39263) 2024-11-16T15:00:18,392 INFO [RS_CLOSE_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=32.95 KB at sequenceid=5 (bloomFilter=false), to=hdfs://localhost:42223/user/jenkins/test-data/a8176cb1-2ebb-b530-b3c7-53c9e53c186e/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/b364a006589fe0d2474ffa45692a6efc/.tmp/colfamily11/947260ab3ca148909c71e7953870e83f 2024-11-16T15:00:18,398 DEBUG [RS_CLOSE_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42223/user/jenkins/test-data/a8176cb1-2ebb-b530-b3c7-53c9e53c186e/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/b364a006589fe0d2474ffa45692a6efc/.tmp/colfamily11/947260ab3ca148909c71e7953870e83f as hdfs://localhost:42223/user/jenkins/test-data/a8176cb1-2ebb-b530-b3c7-53c9e53c186e/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/b364a006589fe0d2474ffa45692a6efc/colfamily11/947260ab3ca148909c71e7953870e83f 2024-11-16T15:00:18,404 INFO [RS_CLOSE_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42223/user/jenkins/test-data/a8176cb1-2ebb-b530-b3c7-53c9e53c186e/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/b364a006589fe0d2474ffa45692a6efc/colfamily11/947260ab3ca148909c71e7953870e83f, entries=703, sequenceid=5, filesize=38.3 K 2024-11-16T15:00:18,405 INFO [RS_CLOSE_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3140): Finished flush of dataSize ~32.95 KB/33744, heapSize ~82.62 KB/84600, currentSize=0 B/0 for b364a006589fe0d2474ffa45692a6efc in 36ms, sequenceid=5, compaction requested=false 2024-11-16T15:00:18,409 DEBUG [RS_CLOSE_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:42223/user/jenkins/test-data/a8176cb1-2ebb-b530-b3c7-53c9e53c186e/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/b364a006589fe0d2474ffa45692a6efc/recovered.edits/8.seqid, newMaxSeqId=8, maxSeqId=1 2024-11-16T15:00:18,410 INFO [RS_CLOSE_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,rrr,1731769026392.b364a006589fe0d2474ffa45692a6efc. 2024-11-16T15:00:18,410 DEBUG [RS_CLOSE_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for b364a006589fe0d2474ffa45692a6efc: Waiting for close lock at 1731769218369Running coprocessor pre-close hooks at 1731769218369Disabling compacts and flushes for region at 1731769218369Disabling writes for close at 1731769218369Obtaining lock to block concurrent updates at 1731769218369Preparing flush snapshotting stores in b364a006589fe0d2474ffa45692a6efc at 1731769218369Finished memstore snapshotting testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,rrr,1731769026392.b364a006589fe0d2474ffa45692a6efc., syncing WAL and waiting on mvcc, flushsize=dataSize=33744, getHeapSize=85080, getOffHeapSize=0, getCellsCount=703 at 1731769218370 (+1 ms)Flushing stores of testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,rrr,1731769026392.b364a006589fe0d2474ffa45692a6efc. at 1731769218371 (+1 ms)Flushing b364a006589fe0d2474ffa45692a6efc/colfamily11: creating writer at 1731769218371Flushing b364a006589fe0d2474ffa45692a6efc/colfamily11: appending metadata at 1731769218385 (+14 ms)Flushing b364a006589fe0d2474ffa45692a6efc/colfamily11: closing flushed file at 1731769218385Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@3b32e99d: reopening flushed file at 1731769218397 (+12 ms)Finished flush of dataSize ~32.95 KB/33744, heapSize ~82.62 KB/84600, currentSize=0 B/0 for b364a006589fe0d2474ffa45692a6efc in 36ms, sequenceid=5, compaction requested=false at 1731769218405 (+8 ms)Writing region close event to WAL at 1731769218406 (+1 ms)Running coprocessor post-close hooks at 1731769218410 (+4 ms)Closed at 1731769218410 2024-11-16T15:00:18,410 DEBUG [RS_CLOSE_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,rrr,1731769026392.b364a006589fe0d2474ffa45692a6efc. 2024-11-16T15:00:18,411 DEBUG [RS_CLOSE_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 266565e60d7419a2a30ceec1b0f6519b, disabling compactions & flushes 2024-11-16T15:00:18,411 INFO [RS_CLOSE_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,lll,1731769026392.266565e60d7419a2a30ceec1b0f6519b. 2024-11-16T15:00:18,411 DEBUG [RS_CLOSE_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,lll,1731769026392.266565e60d7419a2a30ceec1b0f6519b. 2024-11-16T15:00:18,411 DEBUG [RS_CLOSE_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,lll,1731769026392.266565e60d7419a2a30ceec1b0f6519b. after waiting 0 ms 2024-11-16T15:00:18,411 DEBUG [RS_CLOSE_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,lll,1731769026392.266565e60d7419a2a30ceec1b0f6519b. 2024-11-16T15:00:18,411 DEBUG [RS:2;fc3fda29f660:42985 {}] regionserver.HRegionServer(1351): Waiting on 266565e60d7419a2a30ceec1b0f6519b 2024-11-16T15:00:18,411 INFO [RS_CLOSE_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2902): Flushing 266565e60d7419a2a30ceec1b0f6519b 3/3 column families, dataSize=32.95 KB heapSize=83.13 KB 2024-11-16T15:00:18,427 DEBUG [RS_CLOSE_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42223/user/jenkins/test-data/a8176cb1-2ebb-b530-b3c7-53c9e53c186e/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/266565e60d7419a2a30ceec1b0f6519b/.tmp/colfamily11/09ce4c5f315b49089917032e7ebf6b62 is 52, key is lll/colfamily11:colfamily11/1731769030232/Put/seqid=0 2024-11-16T15:00:18,434 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40641 is added to blk_1073741904_1080 (size=39263) 2024-11-16T15:00:18,435 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41671 is added to blk_1073741904_1080 (size=39263) 2024-11-16T15:00:18,435 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36531 is added to blk_1073741904_1080 (size=39263) 2024-11-16T15:00:18,435 INFO [RS_CLOSE_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=32.95 KB at sequenceid=5 (bloomFilter=false), to=hdfs://localhost:42223/user/jenkins/test-data/a8176cb1-2ebb-b530-b3c7-53c9e53c186e/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/266565e60d7419a2a30ceec1b0f6519b/.tmp/colfamily11/09ce4c5f315b49089917032e7ebf6b62 2024-11-16T15:00:18,441 DEBUG [RS_CLOSE_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42223/user/jenkins/test-data/a8176cb1-2ebb-b530-b3c7-53c9e53c186e/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/266565e60d7419a2a30ceec1b0f6519b/.tmp/colfamily11/09ce4c5f315b49089917032e7ebf6b62 as hdfs://localhost:42223/user/jenkins/test-data/a8176cb1-2ebb-b530-b3c7-53c9e53c186e/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/266565e60d7419a2a30ceec1b0f6519b/colfamily11/09ce4c5f315b49089917032e7ebf6b62 2024-11-16T15:00:18,447 INFO [RS_CLOSE_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42223/user/jenkins/test-data/a8176cb1-2ebb-b530-b3c7-53c9e53c186e/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/266565e60d7419a2a30ceec1b0f6519b/colfamily11/09ce4c5f315b49089917032e7ebf6b62, entries=703, sequenceid=5, filesize=38.3 K 2024-11-16T15:00:18,448 INFO [RS_CLOSE_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3140): Finished flush of dataSize ~32.95 KB/33744, heapSize ~82.62 KB/84600, currentSize=0 B/0 for 266565e60d7419a2a30ceec1b0f6519b in 37ms, sequenceid=5, compaction requested=false 2024-11-16T15:00:18,453 DEBUG [RS_CLOSE_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:42223/user/jenkins/test-data/a8176cb1-2ebb-b530-b3c7-53c9e53c186e/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/266565e60d7419a2a30ceec1b0f6519b/recovered.edits/8.seqid, newMaxSeqId=8, maxSeqId=1 2024-11-16T15:00:18,454 INFO [RS_CLOSE_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,lll,1731769026392.266565e60d7419a2a30ceec1b0f6519b. 2024-11-16T15:00:18,454 DEBUG [RS_CLOSE_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 266565e60d7419a2a30ceec1b0f6519b: Waiting for close lock at 1731769218411Running coprocessor pre-close hooks at 1731769218411Disabling compacts and flushes for region at 1731769218411Disabling writes for close at 1731769218411Obtaining lock to block concurrent updates at 1731769218411Preparing flush snapshotting stores in 266565e60d7419a2a30ceec1b0f6519b at 1731769218411Finished memstore snapshotting testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,lll,1731769026392.266565e60d7419a2a30ceec1b0f6519b., syncing WAL and waiting on mvcc, flushsize=dataSize=33744, getHeapSize=85080, getOffHeapSize=0, getCellsCount=703 at 1731769218411Flushing stores of testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,lll,1731769026392.266565e60d7419a2a30ceec1b0f6519b. at 1731769218412 (+1 ms)Flushing 266565e60d7419a2a30ceec1b0f6519b/colfamily11: creating writer at 1731769218412Flushing 266565e60d7419a2a30ceec1b0f6519b/colfamily11: appending metadata at 1731769218427 (+15 ms)Flushing 266565e60d7419a2a30ceec1b0f6519b/colfamily11: closing flushed file at 1731769218427Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@b8bea9c: reopening flushed file at 1731769218440 (+13 ms)Finished flush of dataSize ~32.95 KB/33744, heapSize ~82.62 KB/84600, currentSize=0 B/0 for 266565e60d7419a2a30ceec1b0f6519b in 37ms, sequenceid=5, compaction requested=false at 1731769218448 (+8 ms)Writing region close event to WAL at 1731769218449 (+1 ms)Running coprocessor post-close hooks at 1731769218453 (+4 ms)Closed at 1731769218453 2024-11-16T15:00:18,454 DEBUG [RS_CLOSE_REGION-regionserver/fc3fda29f660:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,lll,1731769026392.266565e60d7419a2a30ceec1b0f6519b. 2024-11-16T15:00:18,611 INFO [RS:2;fc3fda29f660:42985 {}] regionserver.HRegionServer(976): stopping server fc3fda29f660,42985,1731769023951; all regions closed. 2024-11-16T15:00:18,614 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36531 is added to blk_1073741834_1010 (size=353044) 2024-11-16T15:00:18,614 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40641 is added to blk_1073741834_1010 (size=353044) 2024-11-16T15:00:18,614 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41671 is added to blk_1073741834_1010 (size=353044) 2024-11-16T15:00:18,617 DEBUG [RS:2;fc3fda29f660:42985 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/a8176cb1-2ebb-b530-b3c7-53c9e53c186e/oldWALs 2024-11-16T15:00:18,617 INFO [RS:2;fc3fda29f660:42985 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL fc3fda29f660%2C42985%2C1731769023951:(num 1731769025434) 2024-11-16T15:00:18,617 DEBUG [RS:2;fc3fda29f660:42985 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T15:00:18,617 INFO [RS:2;fc3fda29f660:42985 {}] regionserver.LeaseManager(133): Closed leases 2024-11-16T15:00:18,617 INFO [RS:2;fc3fda29f660:42985 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-16T15:00:18,617 INFO [RS:2;fc3fda29f660:42985 {}] hbase.ChoreService(370): Chore service for: regionserver/fc3fda29f660:0 had [ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS] on shutdown 2024-11-16T15:00:18,617 INFO [RS:2;fc3fda29f660:42985 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-16T15:00:18,618 INFO [RS:2;fc3fda29f660:42985 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-16T15:00:18,618 INFO [RS:2;fc3fda29f660:42985 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-16T15:00:18,618 INFO [regionserver/fc3fda29f660:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-16T15:00:18,618 INFO [RS:2;fc3fda29f660:42985 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-16T15:00:18,618 INFO [RS:2;fc3fda29f660:42985 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:42985 2024-11-16T15:00:18,653 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42985-0x101456fc9c70003, quorum=127.0.0.1:63224, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/fc3fda29f660,42985,1731769023951 2024-11-16T15:00:18,653 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43615-0x101456fc9c70000, quorum=127.0.0.1:63224, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-16T15:00:18,653 INFO [RS:2;fc3fda29f660:42985 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-16T15:00:18,655 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [fc3fda29f660,42985,1731769023951] 2024-11-16T15:00:18,669 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/fc3fda29f660,42985,1731769023951 already deleted, retry=false 2024-11-16T15:00:18,669 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; fc3fda29f660,42985,1731769023951 expired; onlineServers=0 2024-11-16T15:00:18,670 INFO [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master 'fc3fda29f660,43615,1731769023046' ***** 2024-11-16T15:00:18,670 INFO [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0 2024-11-16T15:00:18,670 INFO [M:0;fc3fda29f660:43615 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-16T15:00:18,670 INFO [M:0;fc3fda29f660:43615 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-16T15:00:18,670 DEBUG [M:0;fc3fda29f660:43615 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2024-11-16T15:00:18,671 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2024-11-16T15:00:18,671 DEBUG [M:0;fc3fda29f660:43615 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2024-11-16T15:00:18,671 DEBUG [master/fc3fda29f660:0:becomeActiveMaster-HFileCleaner.large.0-1731769025060 {}] cleaner.HFileCleaner(306): Exit Thread[master/fc3fda29f660:0:becomeActiveMaster-HFileCleaner.large.0-1731769025060,5,FailOnTimeoutGroup] 2024-11-16T15:00:18,671 DEBUG [master/fc3fda29f660:0:becomeActiveMaster-HFileCleaner.small.0-1731769025063 {}] cleaner.HFileCleaner(306): Exit Thread[master/fc3fda29f660:0:becomeActiveMaster-HFileCleaner.small.0-1731769025063,5,FailOnTimeoutGroup] 2024-11-16T15:00:18,672 INFO [M:0;fc3fda29f660:43615 {}] hbase.ChoreService(370): Chore service for: master/fc3fda29f660:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown 2024-11-16T15:00:18,672 INFO [M:0;fc3fda29f660:43615 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-16T15:00:18,672 DEBUG [M:0;fc3fda29f660:43615 {}] master.HMaster(1795): Stopping service threads 2024-11-16T15:00:18,672 INFO [M:0;fc3fda29f660:43615 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2024-11-16T15:00:18,673 INFO [M:0;fc3fda29f660:43615 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-16T15:00:18,673 INFO [M:0;fc3fda29f660:43615 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-16T15:00:18,673 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2024-11-16T15:00:18,678 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43615-0x101456fc9c70000, quorum=127.0.0.1:63224, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-11-16T15:00:18,678 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43615-0x101456fc9c70000, quorum=127.0.0.1:63224, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-16T15:00:18,678 DEBUG [M:0;fc3fda29f660:43615 {}] zookeeper.ZKUtil(347): master:43615-0x101456fc9c70000, quorum=127.0.0.1:63224, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error) 2024-11-16T15:00:18,678 WARN [M:0;fc3fda29f660:43615 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null 2024-11-16T15:00:18,680 INFO [M:0;fc3fda29f660:43615 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: hdfs://localhost:42223/user/jenkins/test-data/a8176cb1-2ebb-b530-b3c7-53c9e53c186e/.lastflushedseqids 2024-11-16T15:00:18,698 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40641 is added to blk_1073741905_1081 (size=2837) 2024-11-16T15:00:18,698 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36531 is added to blk_1073741905_1081 (size=2837) 2024-11-16T15:00:18,699 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41671 is added to blk_1073741905_1081 (size=2837) 2024-11-16T15:00:18,699 INFO [M:0;fc3fda29f660:43615 {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-16T15:00:18,699 INFO [M:0;fc3fda29f660:43615 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2024-11-16T15:00:18,699 DEBUG [M:0;fc3fda29f660:43615 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-16T15:00:18,699 INFO [M:0;fc3fda29f660:43615 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-16T15:00:18,699 DEBUG [M:0;fc3fda29f660:43615 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-16T15:00:18,699 DEBUG [M:0;fc3fda29f660:43615 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-16T15:00:18,699 DEBUG [M:0;fc3fda29f660:43615 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-16T15:00:18,700 INFO [M:0;fc3fda29f660:43615 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=344.59 KB heapSize=404.99 KB 2024-11-16T15:00:18,714 DEBUG [M:0;fc3fda29f660:43615 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42223/user/jenkins/test-data/a8176cb1-2ebb-b530-b3c7-53c9e53c186e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/b53b9a3aa47048e099a66da311b652ac is 82, key is hbase:meta,,1/info:regioninfo/1731769212374/Put/seqid=0 2024-11-16T15:00:18,720 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41671 is added to blk_1073741906_1082 (size=6071) 2024-11-16T15:00:18,720 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36531 is added to blk_1073741906_1082 (size=6071) 2024-11-16T15:00:18,721 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40641 is added to blk_1073741906_1082 (size=6071) 2024-11-16T15:00:18,721 INFO [M:0;fc3fda29f660:43615 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1008 B at sequenceid=635 (bloomFilter=true), to=hdfs://localhost:42223/user/jenkins/test-data/a8176cb1-2ebb-b530-b3c7-53c9e53c186e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/b53b9a3aa47048e099a66da311b652ac 2024-11-16T15:00:18,742 DEBUG [M:0;fc3fda29f660:43615 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42223/user/jenkins/test-data/a8176cb1-2ebb-b530-b3c7-53c9e53c186e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/bf551d73733d4900a9d470edb226d5ab is 4107, key is \x00\x00\x00\x00\x00\x00\x00\x04/proc:d/1731769027895/Put/seqid=0 2024-11-16T15:00:18,748 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36531 is added to blk_1073741907_1083 (size=13504) 2024-11-16T15:00:18,748 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41671 is added to blk_1073741907_1083 (size=13504) 2024-11-16T15:00:18,748 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40641 is added to blk_1073741907_1083 (size=13504) 2024-11-16T15:00:18,750 INFO [M:0;fc3fda29f660:43615 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=343.36 KB at sequenceid=635 (bloomFilter=true), to=hdfs://localhost:42223/user/jenkins/test-data/a8176cb1-2ebb-b530-b3c7-53c9e53c186e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/bf551d73733d4900a9d470edb226d5ab 2024-11-16T15:00:18,761 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42985-0x101456fc9c70003, quorum=127.0.0.1:63224, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-16T15:00:18,761 INFO [RS:2;fc3fda29f660:42985 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-16T15:00:18,761 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42985-0x101456fc9c70003, quorum=127.0.0.1:63224, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-16T15:00:18,761 INFO [RS:2;fc3fda29f660:42985 {}] regionserver.HRegionServer(1031): Exiting; stopping=fc3fda29f660,42985,1731769023951; zookeeper connection closed. 2024-11-16T15:00:18,762 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@cad77e0 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@cad77e0 2024-11-16T15:00:18,762 INFO [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 3 regionserver(s) complete 2024-11-16T15:00:18,771 DEBUG [M:0;fc3fda29f660:43615 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42223/user/jenkins/test-data/a8176cb1-2ebb-b530-b3c7-53c9e53c186e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/0f68340e458b4f91ac06b1dda1e10ecd is 69, key is fc3fda29f660,39089,1731769023769/rs:state/1731769025151/Put/seqid=0 2024-11-16T15:00:18,777 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36531 is added to blk_1073741908_1084 (size=5442) 2024-11-16T15:00:18,777 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40641 is added to blk_1073741908_1084 (size=5442) 2024-11-16T15:00:18,778 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41671 is added to blk_1073741908_1084 (size=5442) 2024-11-16T15:00:18,778 INFO [M:0;fc3fda29f660:43615 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=249 B at sequenceid=635 (bloomFilter=true), to=hdfs://localhost:42223/user/jenkins/test-data/a8176cb1-2ebb-b530-b3c7-53c9e53c186e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/0f68340e458b4f91ac06b1dda1e10ecd 2024-11-16T15:00:18,784 INFO [M:0;fc3fda29f660:43615 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 0f68340e458b4f91ac06b1dda1e10ecd 2024-11-16T15:00:18,785 DEBUG [M:0;fc3fda29f660:43615 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42223/user/jenkins/test-data/a8176cb1-2ebb-b530-b3c7-53c9e53c186e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/b53b9a3aa47048e099a66da311b652ac as hdfs://localhost:42223/user/jenkins/test-data/a8176cb1-2ebb-b530-b3c7-53c9e53c186e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/b53b9a3aa47048e099a66da311b652ac 2024-11-16T15:00:18,791 INFO [M:0;fc3fda29f660:43615 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42223/user/jenkins/test-data/a8176cb1-2ebb-b530-b3c7-53c9e53c186e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/b53b9a3aa47048e099a66da311b652ac, entries=14, sequenceid=635, filesize=5.9 K 2024-11-16T15:00:18,792 DEBUG [M:0;fc3fda29f660:43615 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42223/user/jenkins/test-data/a8176cb1-2ebb-b530-b3c7-53c9e53c186e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/bf551d73733d4900a9d470edb226d5ab as hdfs://localhost:42223/user/jenkins/test-data/a8176cb1-2ebb-b530-b3c7-53c9e53c186e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/bf551d73733d4900a9d470edb226d5ab 2024-11-16T15:00:18,797 INFO [M:0;fc3fda29f660:43615 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42223/user/jenkins/test-data/a8176cb1-2ebb-b530-b3c7-53c9e53c186e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/bf551d73733d4900a9d470edb226d5ab, entries=81, sequenceid=635, filesize=13.2 K 2024-11-16T15:00:18,798 DEBUG [M:0;fc3fda29f660:43615 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42223/user/jenkins/test-data/a8176cb1-2ebb-b530-b3c7-53c9e53c186e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/0f68340e458b4f91ac06b1dda1e10ecd as hdfs://localhost:42223/user/jenkins/test-data/a8176cb1-2ebb-b530-b3c7-53c9e53c186e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/0f68340e458b4f91ac06b1dda1e10ecd 2024-11-16T15:00:18,803 INFO [M:0;fc3fda29f660:43615 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 0f68340e458b4f91ac06b1dda1e10ecd 2024-11-16T15:00:18,804 INFO [M:0;fc3fda29f660:43615 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42223/user/jenkins/test-data/a8176cb1-2ebb-b530-b3c7-53c9e53c186e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/0f68340e458b4f91ac06b1dda1e10ecd, entries=3, sequenceid=635, filesize=5.3 K 2024-11-16T15:00:18,805 INFO [M:0;fc3fda29f660:43615 {}] regionserver.HRegion(3140): Finished flush of dataSize ~344.59 KB/352856, heapSize ~404.70 KB/414408, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 105ms, sequenceid=635, compaction requested=false 2024-11-16T15:00:18,806 INFO [M:0;fc3fda29f660:43615 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-16T15:00:18,806 DEBUG [M:0;fc3fda29f660:43615 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1731769218699Disabling compacts and flushes for region at 1731769218699Disabling writes for close at 1731769218699Obtaining lock to block concurrent updates at 1731769218700 (+1 ms)Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1731769218700Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=352856, getHeapSize=414648, getOffHeapSize=0, getCellsCount=800 at 1731769218700Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1731769218700Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1731769218701 (+1 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1731769218714 (+13 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1731769218714Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1731769218726 (+12 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1731769218741 (+15 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1731769218742 (+1 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1731769218754 (+12 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1731769218770 (+16 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1731769218770Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@43e94303: reopening flushed file at 1731769218784 (+14 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5f223aab: reopening flushed file at 1731769218791 (+7 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@51db6c78: reopening flushed file at 1731769218797 (+6 ms)Finished flush of dataSize ~344.59 KB/352856, heapSize ~404.70 KB/414408, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 105ms, sequenceid=635, compaction requested=false at 1731769218805 (+8 ms)Writing region close event to WAL at 1731769218806 (+1 ms)Closed at 1731769218806 2024-11-16T15:00:18,808 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41671 is added to blk_1073741830_1006 (size=395210) 2024-11-16T15:00:18,808 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36531 is added to blk_1073741830_1006 (size=395210) 2024-11-16T15:00:18,808 WARN [Close-WAL-Writer-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(650): complete file /user/jenkins/test-data/a8176cb1-2ebb-b530-b3c7-53c9e53c186e/MasterData/WALs/fc3fda29f660,43615,1731769023046/fc3fda29f660%2C43615%2C1731769023046.1731769024397 not finished, retry = 0 2024-11-16T15:00:18,809 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40641 is added to blk_1073741830_1006 (size=395210) 2024-11-16T15:00:18,910 INFO [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-16T15:00:18,910 INFO [M:0;fc3fda29f660:43615 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down. 2024-11-16T15:00:18,910 INFO [M:0;fc3fda29f660:43615 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:43615 2024-11-16T15:00:18,910 INFO [M:0;fc3fda29f660:43615 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-16T15:00:19,036 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43615-0x101456fc9c70000, quorum=127.0.0.1:63224, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-16T15:00:19,036 INFO [M:0;fc3fda29f660:43615 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-16T15:00:19,036 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43615-0x101456fc9c70000, quorum=127.0.0.1:63224, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-16T15:00:19,095 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@486c2677{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-16T15:00:19,101 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@6738d157{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-16T15:00:19,101 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-16T15:00:19,101 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@43bf9dc1{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-16T15:00:19,101 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@7ed2891b{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2c526703-9314-50ef-2cbc-7a77399c79d8/hadoop.log.dir/,STOPPED} 2024-11-16T15:00:19,103 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-11-16T15:00:19,103 WARN [BP-461342148-172.17.0.2-1731769019248 heartbeating to localhost/127.0.0.1:42223 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-11-16T15:00:19,103 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-11-16T15:00:19,103 WARN [BP-461342148-172.17.0.2-1731769019248 heartbeating to localhost/127.0.0.1:42223 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-461342148-172.17.0.2-1731769019248 (Datanode Uuid 8a23e5b4-97ec-4720-90a3-a377bbd0a053) service to localhost/127.0.0.1:42223 2024-11-16T15:00:19,104 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2c526703-9314-50ef-2cbc-7a77399c79d8/cluster_74004074-8b3a-1fc1-ed3b-d8c79605d2dd/data/data5/current/BP-461342148-172.17.0.2-1731769019248 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-16T15:00:19,105 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2c526703-9314-50ef-2cbc-7a77399c79d8/cluster_74004074-8b3a-1fc1-ed3b-d8c79605d2dd/data/data6/current/BP-461342148-172.17.0.2-1731769019248 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-16T15:00:19,105 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-11-16T15:00:19,107 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@3635c31e{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-16T15:00:19,107 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@78839a64{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-16T15:00:19,107 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-16T15:00:19,107 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@a292a59{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-16T15:00:19,107 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@63779849{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2c526703-9314-50ef-2cbc-7a77399c79d8/hadoop.log.dir/,STOPPED} 2024-11-16T15:00:19,108 WARN [BP-461342148-172.17.0.2-1731769019248 heartbeating to localhost/127.0.0.1:42223 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-11-16T15:00:19,108 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-11-16T15:00:19,108 WARN [BP-461342148-172.17.0.2-1731769019248 heartbeating to localhost/127.0.0.1:42223 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-461342148-172.17.0.2-1731769019248 (Datanode Uuid 3277bc61-d18b-46f8-bf1f-cc24ec81b9e9) service to localhost/127.0.0.1:42223 2024-11-16T15:00:19,108 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-11-16T15:00:19,109 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2c526703-9314-50ef-2cbc-7a77399c79d8/cluster_74004074-8b3a-1fc1-ed3b-d8c79605d2dd/data/data3/current/BP-461342148-172.17.0.2-1731769019248 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-16T15:00:19,109 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2c526703-9314-50ef-2cbc-7a77399c79d8/cluster_74004074-8b3a-1fc1-ed3b-d8c79605d2dd/data/data4/current/BP-461342148-172.17.0.2-1731769019248 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-16T15:00:19,109 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-11-16T15:00:19,111 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@4255463d{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-16T15:00:19,111 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@5e48d68a{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-16T15:00:19,111 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-16T15:00:19,111 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@63d82da9{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-16T15:00:19,111 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@b71ffd0{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2c526703-9314-50ef-2cbc-7a77399c79d8/hadoop.log.dir/,STOPPED} 2024-11-16T15:00:19,112 WARN [BP-461342148-172.17.0.2-1731769019248 heartbeating to localhost/127.0.0.1:42223 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-11-16T15:00:19,112 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-11-16T15:00:19,113 WARN [BP-461342148-172.17.0.2-1731769019248 heartbeating to localhost/127.0.0.1:42223 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-461342148-172.17.0.2-1731769019248 (Datanode Uuid 7b37a58c-887c-4bbb-9985-3e291a925997) service to localhost/127.0.0.1:42223 2024-11-16T15:00:19,113 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-11-16T15:00:19,113 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2c526703-9314-50ef-2cbc-7a77399c79d8/cluster_74004074-8b3a-1fc1-ed3b-d8c79605d2dd/data/data1/current/BP-461342148-172.17.0.2-1731769019248 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-16T15:00:19,113 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2c526703-9314-50ef-2cbc-7a77399c79d8/cluster_74004074-8b3a-1fc1-ed3b-d8c79605d2dd/data/data2/current/BP-461342148-172.17.0.2-1731769019248 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-16T15:00:19,114 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-11-16T15:00:19,126 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@1aa62678{hdfs,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-11-16T15:00:19,127 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@1c0c42ad{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-16T15:00:19,127 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-16T15:00:19,127 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@2a372edd{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-16T15:00:19,127 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@58897554{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2c526703-9314-50ef-2cbc-7a77399c79d8/hadoop.log.dir/,STOPPED} 2024-11-16T15:00:19,138 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers 2024-11-16T15:00:19,202 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down 2024-11-16T15:00:19,210 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: master.procedure.TestSCPWithMeta#testRecoveryAndDoubleExecutionOnRsWithMeta Thread=89 (was 11) Potentially hanging thread: nioEventLoopGroup-2-3 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: SSL Certificates Store Monitor java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) java.base@17.0.11/java.util.TimerThread.mainLoop(Timer.java:537) java.base@17.0.11/java.util.TimerThread.run(Timer.java:516) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-5-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-2-2 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RpcClient-timer-pool-0 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.waitForNextTick(HashedWheelTimer.java:598) app//org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:494) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HMaster-EventLoopGroup-1-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: org.apache.hadoop.hdfs.PeerCache@43b902e5 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hdfs.PeerCache.run(PeerCache.java:253) app//org.apache.hadoop.hdfs.PeerCache.access$000(PeerCache.java:46) app//org.apache.hadoop.hdfs.PeerCache$1.run(PeerCache.java:124) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: region-location-3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1170) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-5-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: master/fc3fda29f660:0:becomeActiveMaster-MemStoreChunkPool Statistics java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Time-limited test.named-queue-events-pool-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) app//com.lmax.disruptor.BlockingWaitStrategy.waitFor(BlockingWaitStrategy.java:47) app//com.lmax.disruptor.ProcessingSequenceBarrier.waitFor(ProcessingSequenceBarrier.java:56) app//com.lmax.disruptor.BatchEventProcessor.processEvents(BatchEventProcessor.java:159) app//com.lmax.disruptor.BatchEventProcessor.run(BatchEventProcessor.java:125) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: master/fc3fda29f660:0:becomeActiveMaster-MemStoreChunkPool Statistics java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: region-location-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1170) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-6-1 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-5-1 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-3-2 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-6-2 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HMaster-EventLoopGroup-1-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:42223 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704) java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903) app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-3-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-7-2 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-4-1 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: regionserver/fc3fda29f660:0.procedureResultReporter java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.RemoteProcedureResultReporter.run(RemoteProcedureResultReporter.java:75) Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:42223 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704) java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903) app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-5-3 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: regionserver/fc3fda29f660:0.procedureResultReporter java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.RemoteProcedureResultReporter.run(RemoteProcedureResultReporter.java:75) Potentially hanging thread: RPCClient-NioEventLoopGroup-6-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-4-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: region-location-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1170) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: org.apache.hadoop.fs.FileSystem$Statistics$StatisticsDataReferenceCleaner java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:155) java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:176) app//org.apache.hadoop.fs.FileSystem$Statistics$StatisticsDataReferenceCleaner.run(FileSystem.java:4171) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Client (608921412) connection to localhost/127.0.0.1:42223 from jenkins.hfs.2 java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) Potentially hanging thread: regionserver/fc3fda29f660:0.Chore.1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-6-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HMaster-EventLoopGroup-1-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Monitor thread for TaskMonitor java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hbase.monitoring.TaskMonitor$MonitorRunnable.run(TaskMonitor.java:325) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-2-1 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: weak-ref-cleaner-strictcontextstorage java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:155) java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:176) app//io.opentelemetry.context.StrictContextStorage$PendingScopes.run(StrictContextStorage.java:269) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: LeaseRenewer:jenkins@localhost:42223 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-5-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-6-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-7-3 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-3-1 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Client (608921412) connection to localhost/127.0.0.1:42223 from jenkins java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) Potentially hanging thread: Timer for 'HBase' metrics system java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.util.TimerThread.mainLoop(Timer.java:563) java.base@17.0.11/java.util.TimerThread.run(Timer.java:516) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-3-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Client (608921412) connection to localhost/127.0.0.1:42223 from jenkins.hfs.0 java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) Potentially hanging thread: Abort regionserver monitor java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.util.TimerThread.mainLoop(Timer.java:563) java.base@17.0.11/java.util.TimerThread.run(Timer.java:516) Potentially hanging thread: HBase-Metrics2-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-4-2 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: SessionTracker java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.zookeeper.server.SessionTrackerImpl.run(SessionTrackerImpl.java:163) Potentially hanging thread: RPCClient-NioEventLoopGroup-6-4 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: region-location-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1170) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: GcTimeMonitor obsWindow = 60000, sleepInterval = 5000, maxGcTimePerc = 100 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.util.GcTimeMonitor.run(GcTimeMonitor.java:161) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-3-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: BootstrapNodeManager java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Client (608921412) connection to localhost/127.0.0.1:42223 from jenkins java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) Potentially hanging thread: ForkJoinPool-2-worker-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkUntil(LockSupport.java:410) java.base@17.0.11/java.util.concurrent.ForkJoinPool.awaitWork(ForkJoinPool.java:1726) java.base@17.0.11/java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1623) java.base@17.0.11/java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:165) Potentially hanging thread: regionserver/fc3fda29f660:0.logRoller java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.hbase.wal.AbstractWALRoller.run(AbstractWALRoller.java:180) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-4-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:42223 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704) java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903) app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-4-3 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_OPEN_META-regionserver/fc3fda29f660:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-6-5 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: SnapshotHandlerChoreCleaner java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_OPEN_REGION-regionserver/fc3fda29f660:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: LeaseRenewer:jenkins.hfs.0@localhost:42223 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-4-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Async-Client-Retry-Timer-pool-0 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.waitForNextTick(HashedWheelTimer.java:598) app//org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:494) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-6-3 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Idle-Rpc-Conn-Sweeper-pool-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: region-location-4 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1170) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-5-2 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-3-3 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: regionserver/fc3fda29f660:0.procedureResultReporter java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.RemoteProcedureResultReporter.run(RemoteProcedureResultReporter.java:75) Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:42223 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704) java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903) app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-6-6 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-7-1 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: LeaseRenewer:jenkins.hfs.2@localhost:42223 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=459 (was 285) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=33 (was 82), ProcessCount=11 (was 11), AvailableMemoryMB=5289 (was 6681)